@oxygen-agent/cli 1.1014.985 → 1.1014.1087
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/command-families/program.d.ts +2 -0
- package/dist/command-families/program.js +241 -0
- package/dist/command-families/tables/automation.d.ts +22 -0
- package/dist/command-families/tables/automation.js +250 -0
- package/dist/command-families/tables/bundle-export.d.ts +7 -0
- package/dist/command-families/tables/bundle-export.js +198 -0
- package/dist/command-families/tables/bundle-format.d.ts +66 -0
- package/dist/command-families/tables/bundle-format.js +274 -0
- package/dist/command-families/tables/bundle-import.d.ts +20 -0
- package/dist/command-families/tables/bundle-import.js +260 -0
- package/dist/command-families/tables/display.d.ts +107 -0
- package/dist/command-families/tables/display.js +229 -0
- package/dist/command-families/tables/import-batches.d.ts +26 -0
- package/dist/command-families/tables/import-batches.js +94 -0
- package/dist/command-families/tables/import-columns.d.ts +24 -0
- package/dist/command-families/tables/import-columns.js +131 -0
- package/dist/command-families/tables/import-upload.d.ts +30 -0
- package/dist/command-families/tables/import-upload.js +203 -0
- package/dist/command-families/tables/import-values.d.ts +10 -0
- package/dist/command-families/tables/import-values.js +60 -0
- package/dist/command-families/tables/imports.d.ts +19 -0
- package/dist/command-families/tables/imports.js +200 -0
- package/dist/command-families/tables/options.d.ts +314 -0
- package/dist/command-families/tables/options.js +1 -0
- package/dist/command-families/tables/register-automation.d.ts +2 -0
- package/dist/command-families/tables/register-automation.js +127 -0
- package/dist/command-families/tables/register-external-imports.d.ts +2 -0
- package/dist/command-families/tables/register-external-imports.js +293 -0
- package/dist/command-families/tables/register-files.d.ts +2 -0
- package/dist/command-families/tables/register-files.js +72 -0
- package/dist/command-families/tables/register-followers.d.ts +2 -0
- package/dist/command-families/tables/register-followers.js +73 -0
- package/dist/command-families/tables/register-lifecycle.d.ts +2 -0
- package/dist/command-families/tables/register-lifecycle.js +114 -0
- package/dist/command-families/tables/register-reads.d.ts +2 -0
- package/dist/command-families/tables/register-reads.js +187 -0
- package/dist/command-families/tables/register-relations.d.ts +2 -0
- package/dist/command-families/tables/register-relations.js +244 -0
- package/dist/command-families/tables/register-row-actions.d.ts +2 -0
- package/dist/command-families/tables/register-row-actions.js +53 -0
- package/dist/command-families/tables/register-row-writes.d.ts +2 -0
- package/dist/command-families/tables/register-row-writes.js +140 -0
- package/dist/command-families/tables/register-views.d.ts +2 -0
- package/dist/command-families/tables/register-views.js +83 -0
- package/dist/command-families/tables/register-watchers.d.ts +2 -0
- package/dist/command-families/tables/register-watchers.js +65 -0
- package/dist/command-families/tables/register-webhooks.d.ts +2 -0
- package/dist/command-families/tables/register-webhooks.js +126 -0
- package/dist/command-families/tables/registration-context.d.ts +8 -0
- package/dist/command-families/tables/registration-context.js +1 -0
- package/dist/command-families/tables/registration.d.ts +3 -0
- package/dist/command-families/tables/registration.js +30 -0
- package/dist/command-families/tables/row-selection.d.ts +5 -0
- package/dist/command-families/tables/row-selection.js +82 -0
- package/dist/command-families/tables/send.d.ts +6 -0
- package/dist/command-families/tables/send.js +111 -0
- package/dist/command-manifest.js +1 -0
- package/dist/commands/copilot.js +8 -0
- package/dist/commands/deliverability.js +4 -2
- package/dist/commands/sequences.js +30 -2
- package/dist/commands/tables.d.ts +1 -3
- package/dist/commands/tables.js +1 -3505
- package/dist/credentials.js +1 -1
- package/dist/customer-operation-commands.js +62 -3
- package/dist/help.js +1 -1
- package/dist/index.d.ts +1 -99
- package/dist/index.js +5 -337
- package/dist/knowledge-mirror.js +1 -1
- package/dist/skills.js +4 -2
- package/dist/update.js +2 -1
- package/dist/workspace-placement-commands.d.ts +3 -0
- package/dist/workspace-placement-commands.js +32 -0
- package/node_modules/@oxygen/shared/dist/agent-sessions.d.ts +4 -0
- package/node_modules/@oxygen/shared/dist/agent-sessions.js +11 -3
- package/node_modules/@oxygen/shared/dist/agents-enabled.d.ts +29 -0
- package/node_modules/@oxygen/shared/dist/agents-enabled.js +51 -0
- package/node_modules/@oxygen/shared/dist/capability-discovery.js +39 -2
- package/node_modules/@oxygen/shared/dist/contracts/agent-sessions.js +1 -1
- package/node_modules/@oxygen/shared/dist/contracts/customer-operations.js +1 -1
- package/node_modules/@oxygen/shared/dist/contracts/guided-activation.js +5 -5
- package/node_modules/@oxygen/shared/dist/contracts/guided-company-files.d.ts +30 -0
- package/node_modules/@oxygen/shared/dist/contracts/guided-company-files.js +53 -0
- package/node_modules/@oxygen/shared/dist/contracts/guided-first-sourcing.d.ts +120 -0
- package/node_modules/@oxygen/shared/dist/contracts/guided-first-sourcing.js +48 -0
- package/node_modules/@oxygen/shared/dist/contracts/guided-onboarding-journey.d.ts +267 -0
- package/node_modules/@oxygen/shared/dist/contracts/guided-onboarding-journey.js +63 -0
- package/node_modules/@oxygen/shared/dist/contracts/operations.d.ts +6 -0
- package/node_modules/@oxygen/shared/dist/contracts/operations.js +9 -4
- package/node_modules/@oxygen/shared/dist/contracts/table-query-raw.d.ts +2 -0
- package/node_modules/@oxygen/shared/dist/contracts/table-query-raw.js +3 -0
- package/node_modules/@oxygen/shared/dist/contracts/teammates.js +17 -15
- package/node_modules/@oxygen/shared/dist/email-tracking-token.d.ts +2 -0
- package/node_modules/@oxygen/shared/dist/email-tracking-token.js +1 -0
- package/node_modules/@oxygen/shared/dist/feature-gates.d.ts +4 -0
- package/node_modules/@oxygen/shared/dist/feature-gates.js +5 -0
- package/node_modules/@oxygen/shared/dist/file-import.d.ts +46 -0
- package/node_modules/@oxygen/shared/dist/file-import.js +138 -0
- package/node_modules/@oxygen/shared/dist/guided-activation.d.ts +8 -0
- package/node_modules/@oxygen/shared/dist/guided-company-files.d.ts +13 -0
- package/node_modules/@oxygen/shared/dist/guided-company-files.js +14 -0
- package/node_modules/@oxygen/shared/dist/guided-first-sourcing.d.ts +116 -0
- package/node_modules/@oxygen/shared/dist/guided-first-sourcing.js +11 -0
- package/node_modules/@oxygen/shared/dist/guided-onboarding-answers.d.ts +4 -0
- package/node_modules/@oxygen/shared/dist/guided-onboarding-answers.js +42 -0
- package/node_modules/@oxygen/shared/dist/guided-onboarding-company-context.d.ts +22 -0
- package/node_modules/@oxygen/shared/dist/guided-onboarding-company-context.js +38 -0
- package/node_modules/@oxygen/shared/dist/guided-onboarding-journey.d.ts +147 -0
- package/node_modules/@oxygen/shared/dist/guided-onboarding-journey.fixture.d.ts +12 -0
- package/node_modules/@oxygen/shared/dist/guided-onboarding-journey.fixture.js +83 -0
- package/node_modules/@oxygen/shared/dist/guided-onboarding-journey.js +44 -0
- package/node_modules/@oxygen/shared/dist/guided-onboarding-prospects.d.ts +7 -0
- package/node_modules/@oxygen/shared/dist/guided-onboarding-prospects.js +13 -0
- package/node_modules/@oxygen/shared/dist/guided-onboarding-signals.d.ts +4 -0
- package/node_modules/@oxygen/shared/dist/guided-onboarding-signals.js +34 -0
- package/node_modules/@oxygen/shared/dist/guided-trial-billing.d.ts +26 -0
- package/node_modules/@oxygen/shared/dist/guided-trial-billing.js +19 -0
- package/node_modules/@oxygen/shared/dist/index.d.ts +0 -1
- package/node_modules/@oxygen/shared/dist/index.js +0 -6
- package/node_modules/@oxygen/shared/dist/table-limits.d.ts +1 -0
- package/node_modules/@oxygen/shared/dist/table-limits.js +6 -0
- package/node_modules/@oxygen/shared/dist/teammates.d.ts +13 -8
- package/node_modules/@oxygen/shared/dist/teammates.js +20 -11
- package/node_modules/@oxygen/shared/dist/user-capability-routing.js +78 -0
- package/node_modules/@oxygen/shared/dist/version.generated.d.ts +1 -1
- package/node_modules/@oxygen/shared/dist/version.generated.js +1 -1
- package/node_modules/@oxygen/shared/dist/workspace-placement-view.d.ts +149 -0
- package/node_modules/@oxygen/shared/dist/workspace-placement-view.js +101 -0
- package/node_modules/@oxygen/shared/dist/workspace-placement.d.ts +77 -0
- package/node_modules/@oxygen/shared/dist/workspace-placement.js +94 -0
- package/package.json +1 -1
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
import { readCount } from "../../command-support.js";
|
|
2
|
+
import { requestOxygen } from "../../http-client.js";
|
|
3
|
+
import { OxygenError } from "@oxygen/shared/cli-result";
|
|
4
|
+
import { MAX_CLI_JSON_BODY_BYTES } from "@oxygen/shared/plan-limits";
|
|
5
|
+
export async function writeImportBatchWithAutoSplit(input) {
|
|
6
|
+
const body = {
|
|
7
|
+
table: input.tableRef,
|
|
8
|
+
rows: input.rows,
|
|
9
|
+
...(input.upsertKey ? { key: input.upsertKey } : {}),
|
|
10
|
+
...(!input.upsertKey && input.requestId ? { request_id: input.requestId } : {}),
|
|
11
|
+
};
|
|
12
|
+
// The server checks content-length against MAX_CLI_JSON_BODY_BYTES and the
|
|
13
|
+
// client sends exactly JSON.stringify(body), so measuring it here is the same
|
|
14
|
+
// test — a wide batch (large text cells) splits before it costs an upload and a
|
|
15
|
+
// 413. Same halving and request_id suffixes as the reactive path below, so a
|
|
16
|
+
// re-run replays the identical batch ids either way.
|
|
17
|
+
const bodyBytes = Buffer.byteLength(JSON.stringify(body));
|
|
18
|
+
if (bodyBytes > MAX_CLI_JSON_BODY_BYTES && input.rows.length > 1) {
|
|
19
|
+
const midpoint = Math.ceil(input.rows.length / 2);
|
|
20
|
+
process.stderr.write(`note: import batch of ${input.rows.length} rows serializes to ${bodyBytes} bytes, `
|
|
21
|
+
+ `over the ${MAX_CLI_JSON_BODY_BYTES} byte request limit; `
|
|
22
|
+
+ `sending as ${midpoint} and ${input.rows.length - midpoint} row batches.\n`);
|
|
23
|
+
const [first, second] = await writeImportBatchHalves(input, midpoint);
|
|
24
|
+
return combineImportBatchWriteSummaries(first, second, { byteSplits: 1 });
|
|
25
|
+
}
|
|
26
|
+
try {
|
|
27
|
+
const result = await requestOxygen(input.upsertKey ? "/api/cli/tables/rows/upsert" : "/api/cli/tables/rows", {
|
|
28
|
+
method: "POST",
|
|
29
|
+
timeoutMs: 300_000,
|
|
30
|
+
body,
|
|
31
|
+
});
|
|
32
|
+
return summarizeImportBatchWrite(result, input.rows.length);
|
|
33
|
+
}
|
|
34
|
+
catch (error) {
|
|
35
|
+
if (!isRequestTooLargeError(error) || input.rows.length <= 1)
|
|
36
|
+
throw error;
|
|
37
|
+
// Fallback for a server whose ceiling is tighter than the shared constant.
|
|
38
|
+
const midpoint = Math.ceil(input.rows.length / 2);
|
|
39
|
+
process.stderr.write(`note: import batch of ${input.rows.length} rows exceeded the request limit; `
|
|
40
|
+
+ `retrying as ${midpoint} and ${input.rows.length - midpoint} row batches.\n`);
|
|
41
|
+
const [first, second] = await writeImportBatchHalves(input, midpoint);
|
|
42
|
+
return combineImportBatchWriteSummaries(first, second, { requestTooLargeRetries: 1 });
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
export async function writeImportBatchHalves(input, midpoint) {
|
|
46
|
+
const first = await writeImportBatchWithAutoSplit({
|
|
47
|
+
...input,
|
|
48
|
+
rows: input.rows.slice(0, midpoint),
|
|
49
|
+
...(input.requestId ? { requestId: `${input.requestId}:0` } : {}),
|
|
50
|
+
});
|
|
51
|
+
const second = await writeImportBatchWithAutoSplit({
|
|
52
|
+
...input,
|
|
53
|
+
rows: input.rows.slice(midpoint),
|
|
54
|
+
...(input.requestId ? { requestId: `${input.requestId}:1` } : {}),
|
|
55
|
+
});
|
|
56
|
+
return [first, second];
|
|
57
|
+
}
|
|
58
|
+
export function summarizeImportBatchWrite(result, batchSize) {
|
|
59
|
+
const warningCount = readCount(result.warningCount);
|
|
60
|
+
const batchWarnings = Array.isArray(result.warnings) ? result.warnings : [];
|
|
61
|
+
return {
|
|
62
|
+
rowCount: readCount(result.rowCount),
|
|
63
|
+
insertedCount: readCount(result.insertedCount),
|
|
64
|
+
updatedCount: readCount(result.updatedCount),
|
|
65
|
+
warningCount,
|
|
66
|
+
warnings: batchWarnings.slice(0, 20),
|
|
67
|
+
warningsTruncated: result.warningsTruncated === true || warningCount > batchWarnings.length,
|
|
68
|
+
writeRequestCount: 1,
|
|
69
|
+
requestTooLargeRetries: 0,
|
|
70
|
+
byteSplits: 0,
|
|
71
|
+
minimumBatchSizeUsed: batchSize,
|
|
72
|
+
lastResult: result,
|
|
73
|
+
};
|
|
74
|
+
}
|
|
75
|
+
export function combineImportBatchWriteSummaries(first, second, extra) {
|
|
76
|
+
const warnings = [...first.warnings, ...second.warnings].slice(0, 20);
|
|
77
|
+
const warningCount = first.warningCount + second.warningCount;
|
|
78
|
+
return {
|
|
79
|
+
rowCount: first.rowCount + second.rowCount,
|
|
80
|
+
insertedCount: first.insertedCount + second.insertedCount,
|
|
81
|
+
updatedCount: first.updatedCount + second.updatedCount,
|
|
82
|
+
warningCount,
|
|
83
|
+
warnings,
|
|
84
|
+
warningsTruncated: first.warningsTruncated || second.warningsTruncated || warningCount > warnings.length,
|
|
85
|
+
writeRequestCount: first.writeRequestCount + second.writeRequestCount,
|
|
86
|
+
requestTooLargeRetries: first.requestTooLargeRetries + second.requestTooLargeRetries + (extra.requestTooLargeRetries ?? 0),
|
|
87
|
+
byteSplits: first.byteSplits + second.byteSplits + (extra.byteSplits ?? 0),
|
|
88
|
+
minimumBatchSizeUsed: Math.min(first.minimumBatchSizeUsed, second.minimumBatchSizeUsed),
|
|
89
|
+
lastResult: second.lastResult ?? first.lastResult,
|
|
90
|
+
};
|
|
91
|
+
}
|
|
92
|
+
export function isRequestTooLargeError(error) {
|
|
93
|
+
return error instanceof OxygenError && error.code === "request_too_large";
|
|
94
|
+
}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import { QueryColumnDescriptor } from "../../command-support.js";
|
|
2
|
+
import { TablesImportOptions } from "./options.js";
|
|
3
|
+
export type ImportTarget = {
|
|
4
|
+
tableRef: string;
|
|
5
|
+
rows: Record<string, unknown>[];
|
|
6
|
+
createdTable: unknown | null;
|
|
7
|
+
tableWebUrl: string | null;
|
|
8
|
+
upsertKey: string | undefined;
|
|
9
|
+
sourceKeyMap: Record<string, string> | null;
|
|
10
|
+
};
|
|
11
|
+
export declare function prepareImportTarget(table: string | undefined, options: TablesImportOptions, parsedRows: Record<string, unknown>[]): Promise<ImportTarget>;
|
|
12
|
+
export declare function resolveExistingTableImportKeyMapping(table: string, rows: Record<string, unknown>[], upsertKey: string | undefined): Promise<{
|
|
13
|
+
sourceKeyMap: Record<string, string> | null;
|
|
14
|
+
upsertKey: string | undefined;
|
|
15
|
+
}>;
|
|
16
|
+
export type ExistingTableColumnResolver = {
|
|
17
|
+
exact: Map<string, string>;
|
|
18
|
+
aliases: Map<string, string | null>;
|
|
19
|
+
};
|
|
20
|
+
export declare function buildExistingTableColumnResolver(columns: QueryColumnDescriptor[]): ExistingTableColumnResolver;
|
|
21
|
+
export declare function addImportColumnAlias(aliases: Map<string, string | null>, alias: string | undefined | null, key: string): void;
|
|
22
|
+
export declare function resolveImportSourceKey(sourceKey: string, resolver: ExistingTableColumnResolver): string | null;
|
|
23
|
+
export declare function readImportColumnAlias(aliases: Map<string, string | null>, alias: string): string | null;
|
|
24
|
+
export declare function remapImportRows(rows: Record<string, unknown>[], sourceKeyMap: Record<string, string> | null): Record<string, unknown>[];
|
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
import { readRecordString } from "../../cli-values.js";
|
|
2
|
+
import { readRecord } from "../../command-support.js";
|
|
3
|
+
import { requestOxygen } from "../../http-client.js";
|
|
4
|
+
import { readOption } from "../../util.js";
|
|
5
|
+
import { OxygenError } from "@oxygen/shared/cli-result";
|
|
6
|
+
import { normalizeRowsForNewTable, inferImportColumnLabels, normalizeImportColumnKey } from "@oxygen/shared/file-import";
|
|
7
|
+
import { tableWebUrl } from "./import-values.js";
|
|
8
|
+
import { normalizeCreatedTableUpsertKey } from "./bundle-format.js";
|
|
9
|
+
export async function prepareImportTarget(table, options, parsedRows) {
|
|
10
|
+
if (options.create && table) {
|
|
11
|
+
throw new OxygenError("invalid_import_target", "Pass either a table argument or --create, not both.", {
|
|
12
|
+
exitCode: 1,
|
|
13
|
+
});
|
|
14
|
+
}
|
|
15
|
+
if (!options.create && !table) {
|
|
16
|
+
throw new OxygenError("invalid_import_target", "Pass a table argument or --create <name>.", {
|
|
17
|
+
exitCode: 1,
|
|
18
|
+
});
|
|
19
|
+
}
|
|
20
|
+
if (!options.create) {
|
|
21
|
+
const keyMapping = await resolveExistingTableImportKeyMapping(table, parsedRows, readOption(options.upsertKey) ?? undefined);
|
|
22
|
+
return {
|
|
23
|
+
tableRef: table,
|
|
24
|
+
rows: remapImportRows(parsedRows, keyMapping.sourceKeyMap),
|
|
25
|
+
createdTable: null,
|
|
26
|
+
tableWebUrl: null,
|
|
27
|
+
upsertKey: keyMapping.upsertKey,
|
|
28
|
+
sourceKeyMap: keyMapping.sourceKeyMap,
|
|
29
|
+
};
|
|
30
|
+
}
|
|
31
|
+
const normalized = normalizeRowsForNewTable(parsedRows);
|
|
32
|
+
const created = await requestOxygen("/api/cli/tables", {
|
|
33
|
+
method: "POST",
|
|
34
|
+
body: {
|
|
35
|
+
name: options.create,
|
|
36
|
+
columns: normalized.columns,
|
|
37
|
+
...(readOption(options.project) ? { project: readOption(options.project) } : {}),
|
|
38
|
+
},
|
|
39
|
+
});
|
|
40
|
+
const tableRecord = readRecord(created, "table");
|
|
41
|
+
const createdSlug = readRecordString(tableRecord, "slug") ?? readRecordString(tableRecord, "id");
|
|
42
|
+
if (!createdSlug) {
|
|
43
|
+
throw new OxygenError("invalid_response", "Oxygen API response is missing the created table slug.", {
|
|
44
|
+
exitCode: 1,
|
|
45
|
+
});
|
|
46
|
+
}
|
|
47
|
+
const createdWebUrl = readRecordString(created, "web_url")
|
|
48
|
+
?? readRecordString(created, "deepLink");
|
|
49
|
+
return {
|
|
50
|
+
tableRef: createdSlug,
|
|
51
|
+
rows: normalized.rows,
|
|
52
|
+
createdTable: created,
|
|
53
|
+
tableWebUrl: createdWebUrl ?? tableWebUrl(createdSlug),
|
|
54
|
+
upsertKey: normalizeCreatedTableUpsertKey(options.upsertKey, normalized.keyBySource),
|
|
55
|
+
sourceKeyMap: normalized.keyBySource,
|
|
56
|
+
};
|
|
57
|
+
}
|
|
58
|
+
export async function resolveExistingTableImportKeyMapping(table, rows, upsertKey) {
|
|
59
|
+
const describe = await requestOxygen("/api/cli/tables/describe", { method: "POST", body: { table } });
|
|
60
|
+
const resolver = buildExistingTableColumnResolver(describe.columns ?? []);
|
|
61
|
+
const resolvedByHeader = new Map();
|
|
62
|
+
const targetCounts = new Map();
|
|
63
|
+
for (const sourceKey of inferImportColumnLabels(rows)) {
|
|
64
|
+
const resolved = resolveImportSourceKey(sourceKey, resolver);
|
|
65
|
+
if (!resolved)
|
|
66
|
+
continue;
|
|
67
|
+
resolvedByHeader.set(sourceKey, resolved);
|
|
68
|
+
targetCounts.set(resolved, (targetCounts.get(resolved) ?? 0) + 1);
|
|
69
|
+
}
|
|
70
|
+
const sourceKeyMap = {};
|
|
71
|
+
for (const [sourceKey, resolved] of resolvedByHeader) {
|
|
72
|
+
if ((targetCounts.get(resolved) ?? 0) > 1)
|
|
73
|
+
continue;
|
|
74
|
+
if (resolved !== sourceKey)
|
|
75
|
+
sourceKeyMap[sourceKey] = resolved;
|
|
76
|
+
}
|
|
77
|
+
return {
|
|
78
|
+
sourceKeyMap: Object.keys(sourceKeyMap).length > 0 ? sourceKeyMap : null,
|
|
79
|
+
upsertKey: upsertKey
|
|
80
|
+
? resolveImportSourceKey(upsertKey, resolver) ?? upsertKey
|
|
81
|
+
: undefined,
|
|
82
|
+
};
|
|
83
|
+
}
|
|
84
|
+
export function buildExistingTableColumnResolver(columns) {
|
|
85
|
+
const exact = new Map();
|
|
86
|
+
const aliases = new Map();
|
|
87
|
+
for (const column of columns) {
|
|
88
|
+
if (!column.key)
|
|
89
|
+
continue;
|
|
90
|
+
exact.set(column.key, column.key);
|
|
91
|
+
addImportColumnAlias(aliases, column.label, column.key);
|
|
92
|
+
addImportColumnAlias(aliases, normalizeImportColumnKey(column.key), column.key);
|
|
93
|
+
if (column.label)
|
|
94
|
+
addImportColumnAlias(aliases, normalizeImportColumnKey(column.label), column.key);
|
|
95
|
+
}
|
|
96
|
+
return { exact, aliases };
|
|
97
|
+
}
|
|
98
|
+
export function addImportColumnAlias(aliases, alias, key) {
|
|
99
|
+
const normalizedAlias = alias?.trim();
|
|
100
|
+
if (!normalizedAlias)
|
|
101
|
+
return;
|
|
102
|
+
const existing = aliases.get(normalizedAlias);
|
|
103
|
+
if (existing === undefined) {
|
|
104
|
+
aliases.set(normalizedAlias, key);
|
|
105
|
+
return;
|
|
106
|
+
}
|
|
107
|
+
if (existing !== key)
|
|
108
|
+
aliases.set(normalizedAlias, null);
|
|
109
|
+
}
|
|
110
|
+
export function resolveImportSourceKey(sourceKey, resolver) {
|
|
111
|
+
const trimmed = sourceKey.trim();
|
|
112
|
+
return resolver.exact.get(sourceKey)
|
|
113
|
+
?? resolver.exact.get(trimmed)
|
|
114
|
+
?? readImportColumnAlias(resolver.aliases, sourceKey)
|
|
115
|
+
?? readImportColumnAlias(resolver.aliases, trimmed)
|
|
116
|
+
?? readImportColumnAlias(resolver.aliases, normalizeImportColumnKey(sourceKey));
|
|
117
|
+
}
|
|
118
|
+
export function readImportColumnAlias(aliases, alias) {
|
|
119
|
+
return aliases.get(alias) ?? null;
|
|
120
|
+
}
|
|
121
|
+
export function remapImportRows(rows, sourceKeyMap) {
|
|
122
|
+
if (!sourceKeyMap)
|
|
123
|
+
return rows;
|
|
124
|
+
return rows.map((row) => {
|
|
125
|
+
const remapped = {};
|
|
126
|
+
for (const [key, value] of Object.entries(row)) {
|
|
127
|
+
remapped[sourceKeyMap[key] ?? key] = value;
|
|
128
|
+
}
|
|
129
|
+
return remapped;
|
|
130
|
+
});
|
|
131
|
+
}
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
import type { RowsFileFormat } from "@oxygen/shared/file-import";
|
|
2
|
+
import { TablesFileImportOptions } from "./options.js";
|
|
3
|
+
import { ImportTarget } from "./import-columns.js";
|
|
4
|
+
/**
|
|
5
|
+
* The file a staged import uploads. `read` supplies the rows its target is
|
|
6
|
+
* prepared from, the exact row count and the digest; it runs AFTER the presign,
|
|
7
|
+
* so a file over the plan's limit is refused before a large file is scanned.
|
|
8
|
+
*/
|
|
9
|
+
export type StagedImportFile = {
|
|
10
|
+
byteLength: number;
|
|
11
|
+
read: () => Promise<{
|
|
12
|
+
targetRows: Record<string, unknown>[];
|
|
13
|
+
rowCount: number;
|
|
14
|
+
sha256: string;
|
|
15
|
+
}>;
|
|
16
|
+
put: (uploadUrl: string, contentType: string) => Promise<Response>;
|
|
17
|
+
};
|
|
18
|
+
export declare function tryEnqueueStagedFileImport(// skipcq: JS-R1005
|
|
19
|
+
table: string | undefined, options: TablesFileImportOptions, format: RowsFileFormat, file: StagedImportFile, context: {
|
|
20
|
+
autoBackground: boolean;
|
|
21
|
+
batchSize: number;
|
|
22
|
+
}, preparedTarget?: ImportTarget | null): Promise<Record<string, unknown> | null>;
|
|
23
|
+
export declare function formatMebibytes(bytes: number): string;
|
|
24
|
+
export declare function putImportFile(uploadUrl: string, fileBuffer: Buffer, contentType: string): Promise<Response>;
|
|
25
|
+
export declare function importFileContentType(format: RowsFileFormat): string;
|
|
26
|
+
export declare function enqueueImportFile(table: string | undefined, options: TablesFileImportOptions, format: RowsFileFormat, batchSize: number, context: {
|
|
27
|
+
autoBackground: boolean;
|
|
28
|
+
sourceHash: string;
|
|
29
|
+
}, preparedTarget?: ImportTarget | null): Promise<Record<string, unknown>>;
|
|
30
|
+
export declare function emitQueueWaitStderrNote(queueWait: Record<string, unknown> | null): void;
|
|
@@ -0,0 +1,203 @@
|
|
|
1
|
+
import { readRecordString, readPositiveInt } from "../../cli-values.js";
|
|
2
|
+
import { readRecord } from "../../command-support.js";
|
|
3
|
+
import { requestOxygen } from "../../http-client.js";
|
|
4
|
+
import { readOption } from "../../util.js";
|
|
5
|
+
import { OxygenError } from "@oxygen/shared/cli-result";
|
|
6
|
+
import { randomUUID } from "node:crypto";
|
|
7
|
+
import { readFileSync } from "node:fs";
|
|
8
|
+
import { basename } from "node:path";
|
|
9
|
+
import { prepareImportTarget } from "./import-columns.js";
|
|
10
|
+
export async function tryEnqueueStagedFileImport(// skipcq: JS-R1005
|
|
11
|
+
table, options, format, file, context, preparedTarget = null) {
|
|
12
|
+
if (options.create && table) {
|
|
13
|
+
throw new OxygenError("invalid_import_target", "Pass either a table argument or --create, not both.", {
|
|
14
|
+
exitCode: 1,
|
|
15
|
+
});
|
|
16
|
+
}
|
|
17
|
+
if (!options.create && !table) {
|
|
18
|
+
throw new OxygenError("invalid_import_target", "Pass a table argument or --create <name>.", {
|
|
19
|
+
exitCode: 1,
|
|
20
|
+
});
|
|
21
|
+
}
|
|
22
|
+
const filename = basename(options.file);
|
|
23
|
+
const contentType = importFileContentType(format);
|
|
24
|
+
const traceId = randomUUID();
|
|
25
|
+
// Probe for a presigned upload URL. If object storage isn't configured the
|
|
26
|
+
// server returns object_storage_not_configured; signal the caller to fall
|
|
27
|
+
// back to the inline multipart import by returning null.
|
|
28
|
+
const requestPresign = async () => {
|
|
29
|
+
try {
|
|
30
|
+
return await requestOxygen("/api/cli/tables/import-url", {
|
|
31
|
+
method: "POST",
|
|
32
|
+
body: {
|
|
33
|
+
file_name: filename,
|
|
34
|
+
content_type: contentType,
|
|
35
|
+
byte_length: file.byteLength,
|
|
36
|
+
format,
|
|
37
|
+
trace_id: traceId,
|
|
38
|
+
},
|
|
39
|
+
});
|
|
40
|
+
}
|
|
41
|
+
catch (error) {
|
|
42
|
+
if (error instanceof OxygenError && error.code === "object_storage_not_configured") {
|
|
43
|
+
return null;
|
|
44
|
+
}
|
|
45
|
+
throw error;
|
|
46
|
+
}
|
|
47
|
+
};
|
|
48
|
+
let presigned = await requestPresign();
|
|
49
|
+
if (!presigned)
|
|
50
|
+
return null;
|
|
51
|
+
const contents = await file.read();
|
|
52
|
+
// Create the table for --create (or resolve the existing ref) before the
|
|
53
|
+
// upload so a presign success always pairs with a real target.
|
|
54
|
+
const target = preparedTarget ?? await prepareImportTarget(table, options, contents.targetRows);
|
|
55
|
+
const put = (current) => {
|
|
56
|
+
const uploadUrl = readRecordString(current, "upload_url");
|
|
57
|
+
return uploadUrl ? file.put(uploadUrl, contentType) : Promise.resolve(new Response(null, { status: 500 }));
|
|
58
|
+
};
|
|
59
|
+
let putResponse = await put(presigned);
|
|
60
|
+
// A direct-upload 403 most commonly means a stale/mismatched signature.
|
|
61
|
+
// Refresh once under the same client trace so a transient presign never
|
|
62
|
+
// strands a durable import, while persistent policy failures stay explicit.
|
|
63
|
+
if (putResponse.status === 403) {
|
|
64
|
+
const refreshed = await requestPresign();
|
|
65
|
+
if (refreshed) {
|
|
66
|
+
presigned = refreshed;
|
|
67
|
+
putResponse = await put(presigned);
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
if (!putResponse.ok) {
|
|
71
|
+
throw new OxygenError("import_upload_failed", `Uploading the import file to object storage failed (HTTP ${putResponse.status}).`, { details: { status: putResponse.status, trace_id: traceId }, exitCode: 1 });
|
|
72
|
+
}
|
|
73
|
+
const storageKey = readRecordString(presigned, "storage_key");
|
|
74
|
+
const storageBucket = readRecordString(presigned, "storage_bucket");
|
|
75
|
+
const storageProvider = readRecordString(presigned, "storage_provider") ?? "s3";
|
|
76
|
+
if (!storageKey)
|
|
77
|
+
return null;
|
|
78
|
+
const result = await requestOxygen("/api/cli/tables/import-staged", {
|
|
79
|
+
method: "POST",
|
|
80
|
+
timeoutMs: 120_000,
|
|
81
|
+
body: {
|
|
82
|
+
table: target.tableRef,
|
|
83
|
+
storage_provider: storageProvider,
|
|
84
|
+
storage_bucket: storageBucket,
|
|
85
|
+
storage_key: storageKey,
|
|
86
|
+
format,
|
|
87
|
+
file_name: filename,
|
|
88
|
+
byte_length: file.byteLength,
|
|
89
|
+
sha256: contents.sha256,
|
|
90
|
+
row_count: contents.rowCount,
|
|
91
|
+
batch_size: context.batchSize,
|
|
92
|
+
...(readPositiveInt(options.maxConcurrency)
|
|
93
|
+
? { max_concurrency: readPositiveInt(options.maxConcurrency) }
|
|
94
|
+
: {}),
|
|
95
|
+
...(target.upsertKey ? { upsert_key: target.upsertKey } : {}),
|
|
96
|
+
...(target.sourceKeyMap ? { source_key_map: target.sourceKeyMap } : {}),
|
|
97
|
+
...(options.skipBlankCells ? { skip_blank_cells: true } : {}),
|
|
98
|
+
},
|
|
99
|
+
});
|
|
100
|
+
emitQueueWaitStderrNote(readRecord(result, "queue_wait"));
|
|
101
|
+
return {
|
|
102
|
+
...result,
|
|
103
|
+
...(target.createdTable ? { createdTable: target.createdTable } : {}),
|
|
104
|
+
...(target.tableWebUrl ? { table_web_url: target.tableWebUrl } : {}),
|
|
105
|
+
background: true,
|
|
106
|
+
autoBackground: context.autoBackground,
|
|
107
|
+
import_engine: "chunked_file_v2",
|
|
108
|
+
storage_provider: storageProvider,
|
|
109
|
+
};
|
|
110
|
+
}
|
|
111
|
+
export function formatMebibytes(bytes) {
|
|
112
|
+
return `${Math.round(bytes / (1024 * 1024)).toLocaleString("en-US")} MiB`;
|
|
113
|
+
}
|
|
114
|
+
export async function putImportFile(uploadUrl, fileBuffer, contentType) {
|
|
115
|
+
const controller = new AbortController();
|
|
116
|
+
const timer = setTimeout(() => controller.abort(), 300_000);
|
|
117
|
+
try {
|
|
118
|
+
return await fetch(uploadUrl, {
|
|
119
|
+
method: "PUT",
|
|
120
|
+
headers: {
|
|
121
|
+
"content-type": contentType,
|
|
122
|
+
"content-length": String(fileBuffer.byteLength),
|
|
123
|
+
},
|
|
124
|
+
body: new Uint8Array(fileBuffer),
|
|
125
|
+
signal: controller.signal,
|
|
126
|
+
});
|
|
127
|
+
}
|
|
128
|
+
finally {
|
|
129
|
+
clearTimeout(timer);
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
export function importFileContentType(format) {
|
|
133
|
+
if (format === "csv")
|
|
134
|
+
return "text/csv";
|
|
135
|
+
if (format === "xlsx")
|
|
136
|
+
return "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet";
|
|
137
|
+
if (format === "jsonl")
|
|
138
|
+
return "application/x-ndjson";
|
|
139
|
+
return "application/json";
|
|
140
|
+
}
|
|
141
|
+
export async function enqueueImportFile(table, options, format, batchSize, context, preparedTarget = null) {
|
|
142
|
+
if (options.create && table) {
|
|
143
|
+
throw new OxygenError("invalid_import_target", "Pass either a table argument or --create, not both.", {
|
|
144
|
+
exitCode: 1,
|
|
145
|
+
});
|
|
146
|
+
}
|
|
147
|
+
if (!options.create && !table) {
|
|
148
|
+
throw new OxygenError("invalid_import_target", "Pass a table argument or --create <name>.", {
|
|
149
|
+
exitCode: 1,
|
|
150
|
+
});
|
|
151
|
+
}
|
|
152
|
+
if (options.skipBlankCells) {
|
|
153
|
+
// The multipart import cannot leave blank cells out; failing beats clearing them.
|
|
154
|
+
throw new OxygenError("skip_blank_cells_unavailable", "--skip-blank-cells needs the staged file upload, which this OXYGEN server has not configured. Add --sync to write the rows from this machine instead.", { exitCode: 1 });
|
|
155
|
+
}
|
|
156
|
+
const fileBuffer = readFileSync(options.file);
|
|
157
|
+
const filename = basename(options.file);
|
|
158
|
+
const form = new FormData();
|
|
159
|
+
const file = new File([fileBuffer], filename);
|
|
160
|
+
form.append("file", file, filename);
|
|
161
|
+
form.append("filename", filename);
|
|
162
|
+
form.append("format", format);
|
|
163
|
+
form.append("batch_size", String(batchSize));
|
|
164
|
+
form.append("auto_background", String(context.autoBackground));
|
|
165
|
+
form.append("source_hash", context.sourceHash);
|
|
166
|
+
const targetTable = preparedTarget?.tableRef ?? table;
|
|
167
|
+
if (targetTable)
|
|
168
|
+
form.append("table", targetTable);
|
|
169
|
+
if (!preparedTarget && options.create)
|
|
170
|
+
form.append("table_name", options.create);
|
|
171
|
+
const project = readOption(options.project);
|
|
172
|
+
if (project)
|
|
173
|
+
form.append("project", project);
|
|
174
|
+
const upsertKey = preparedTarget?.upsertKey ?? readOption(options.upsertKey);
|
|
175
|
+
if (upsertKey)
|
|
176
|
+
form.append("upsert_key", upsertKey);
|
|
177
|
+
if (preparedTarget?.sourceKeyMap) {
|
|
178
|
+
form.append("source_key_map", JSON.stringify(preparedTarget.sourceKeyMap));
|
|
179
|
+
}
|
|
180
|
+
if (options.sheet)
|
|
181
|
+
form.append("sheet", options.sheet);
|
|
182
|
+
const maxConcurrency = readPositiveInt(options.maxConcurrency);
|
|
183
|
+
if (maxConcurrency)
|
|
184
|
+
form.append("max_concurrency", String(maxConcurrency));
|
|
185
|
+
const result = await requestOxygen("/api/cli/tables/import-file", {
|
|
186
|
+
method: "POST",
|
|
187
|
+
timeoutMs: 300_000,
|
|
188
|
+
formData: form,
|
|
189
|
+
});
|
|
190
|
+
emitQueueWaitStderrNote(readRecord(result, "queue_wait"));
|
|
191
|
+
return {
|
|
192
|
+
...result,
|
|
193
|
+
background: true,
|
|
194
|
+
autoBackground: context.autoBackground,
|
|
195
|
+
};
|
|
196
|
+
}
|
|
197
|
+
export function emitQueueWaitStderrNote(queueWait) {
|
|
198
|
+
if (!queueWait)
|
|
199
|
+
return;
|
|
200
|
+
const note = readRecordString(queueWait, "note");
|
|
201
|
+
if (note)
|
|
202
|
+
process.stderr.write(`note: ${note}\n`);
|
|
203
|
+
}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
export declare const LARGE_IMPORT_BACKGROUND_ROW_THRESHOLD = 500;
|
|
2
|
+
export declare const SAFE_IMPORT_WRITE_BATCH_SIZE = 500;
|
|
3
|
+
export declare const IMPORT_FILE_LIMIT_HELP: string;
|
|
4
|
+
export declare function tableWebUrl(tableIdOrSlug: string): string;
|
|
5
|
+
export declare function formatImportFileSizeLimit(tier: "free" | "starter" | "pro" | "scale"): string;
|
|
6
|
+
export declare function formatImportByteSize(bytes: number): string;
|
|
7
|
+
export declare function formatJsonBodyLimit(): string;
|
|
8
|
+
export declare function withImportWaitNextStep(envelope: Record<string, unknown>): Record<string, unknown>;
|
|
9
|
+
export declare function normalizeImportBatchSize(value: string | undefined): number;
|
|
10
|
+
export declare function hashImportFile(path: string): string;
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
import { readRecordString, readPositiveInt } from "../../cli-values.js";
|
|
2
|
+
import { readRecord } from "../../command-support.js";
|
|
3
|
+
import { defaultApiUrl } from "../../credentials.js";
|
|
4
|
+
import { oxygenAppUrl } from "../../help.js";
|
|
5
|
+
import { STREAMED_FILE_IMPORT_MIN_BYTES } from "../../streamed-file-import.js";
|
|
6
|
+
import { OxygenError } from "@oxygen/shared/cli-result";
|
|
7
|
+
import { MAX_BUFFERED_IMPORT_PARSE_BYTES } from "@oxygen/shared/file-import";
|
|
8
|
+
import { PLAN_LIMITS, MAX_CLI_JSON_BODY_BYTES } from "@oxygen/shared/plan-limits";
|
|
9
|
+
import { createHash } from "node:crypto";
|
|
10
|
+
import { readFileSync } from "node:fs";
|
|
11
|
+
export const LARGE_IMPORT_BACKGROUND_ROW_THRESHOLD = 500;
|
|
12
|
+
export const SAFE_IMPORT_WRITE_BATCH_SIZE = 500;
|
|
13
|
+
export const IMPORT_FILE_LIMIT_HELP = `Per-file row limit: your plan's rows-per-Table limit, since one file can fill an empty Table (\`oxygen limits show\` lists it for every plan size). File-size limit: ${formatImportFileSizeLimit("free")} on free, ${formatImportFileSizeLimit("starter")} on $49-$99, ${formatImportFileSizeLimit("pro")} on $199-$499, ${formatImportFileSizeLimit("scale")} on $999 and up. JSON and XLSX are read whole, so they stop at ${formatImportByteSize(MAX_BUFFERED_IMPORT_PARSE_BYTES)}; save a larger file as CSV or JSONL. Files over ${formatImportByteSize(STREAMED_FILE_IMPORT_MIN_BYTES)} upload straight from disk and always load in the background.`;
|
|
14
|
+
// Deep-links must name the host the command actually talked to. This hardcoded
|
|
15
|
+
// https://oxygen-agent.com, so every `oxygen-dev` bundle export/import handed
|
|
16
|
+
// back a PRODUCTION url for a table that only exists on dev (OXY-4360).
|
|
17
|
+
// `defaultApiUrl()` is the same resolver the request layer uses, so the link and
|
|
18
|
+
// the call can no longer disagree.
|
|
19
|
+
export function tableWebUrl(tableIdOrSlug) {
|
|
20
|
+
return oxygenAppUrl(defaultApiUrl(), `/tables/${encodeURIComponent(tableIdOrSlug)}`);
|
|
21
|
+
}
|
|
22
|
+
export function formatImportFileSizeLimit(tier) {
|
|
23
|
+
return formatImportByteSize(PLAN_LIMITS[tier].import.maxFileBytes);
|
|
24
|
+
}
|
|
25
|
+
export function formatImportByteSize(bytes) {
|
|
26
|
+
const mebibytes = bytes / (1024 * 1024);
|
|
27
|
+
return mebibytes >= 1024 && mebibytes % 1024 === 0 ? `${mebibytes / 1024} GiB` : `${Math.round(mebibytes)} MiB`;
|
|
28
|
+
}
|
|
29
|
+
// Help copy reads the shared ceiling so the number can never drift from the
|
|
30
|
+
// one the server enforces and the import writer splits against.
|
|
31
|
+
export function formatJsonBodyLimit() {
|
|
32
|
+
return `${Math.round(MAX_CLI_JSON_BODY_BYTES / 1_000_000)} MB`;
|
|
33
|
+
}
|
|
34
|
+
// A backgrounded import returns as soon as the file is staged, with counts.rows
|
|
35
|
+
// still 0 while the worker loads. With no follow-up command in the envelope that
|
|
36
|
+
// reads as a truncated import - which is how a 500-row `--batch-size` default
|
|
37
|
+
// gets mistaken for a 500-row cap. `table-ingestions wait` takes the ingestion
|
|
38
|
+
// run's own id (`ingestionRunId`), not the linked `runId`, which 404s.
|
|
39
|
+
export function withImportWaitNextStep(envelope) {
|
|
40
|
+
const ingestionRunId = readRecordString(envelope, "ingestionRunId")
|
|
41
|
+
?? readRecordString(readRecord(envelope, "ingestionRun"), "id");
|
|
42
|
+
if (!ingestionRunId)
|
|
43
|
+
return envelope;
|
|
44
|
+
const nextStep = `oxygen table-ingestions wait ${ingestionRunId}`;
|
|
45
|
+
process.stderr.write(`note: rows load in the background - watch with ${nextStep}\n`);
|
|
46
|
+
return { ...envelope, next_step: nextStep };
|
|
47
|
+
}
|
|
48
|
+
export function normalizeImportBatchSize(value) {
|
|
49
|
+
const batchSize = readPositiveInt(value) ?? 500;
|
|
50
|
+
if (batchSize > 5000) {
|
|
51
|
+
throw new OxygenError("invalid_number", "Import batch size cannot exceed 5000.", {
|
|
52
|
+
details: { value },
|
|
53
|
+
exitCode: 1,
|
|
54
|
+
});
|
|
55
|
+
}
|
|
56
|
+
return batchSize;
|
|
57
|
+
}
|
|
58
|
+
export function hashImportFile(path) {
|
|
59
|
+
return createHash("sha256").update(readFileSync(path)).digest("hex");
|
|
60
|
+
}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import type { RowsFileFormat } from "@oxygen/shared/file-import";
|
|
2
|
+
import { TablesFileImportOptions, TablesImportOptions } from "./options.js";
|
|
3
|
+
/**
|
|
4
|
+
* `--url` hands the fetch to the server: the same durable import the file path
|
|
5
|
+
* enqueues, with the bytes read by OXYGEN behind SSRF, size and content guards,
|
|
6
|
+
* so the CLI never downloads anything and a 0-credit preview is one flag away.
|
|
7
|
+
*/
|
|
8
|
+
export declare function importRowsFromUrl(table: string | undefined, options: TablesImportOptions & {
|
|
9
|
+
url: string;
|
|
10
|
+
}): Promise<Record<string, unknown>>;
|
|
11
|
+
export declare function importRows(table: string | undefined, options: TablesImportOptions): Promise<Record<string, unknown>>;
|
|
12
|
+
export declare function importRowsFromFile(table: string | undefined, options: TablesFileImportOptions): Promise<Record<string, unknown>>;
|
|
13
|
+
/**
|
|
14
|
+
* A CSV or JSONL file over STREAMED_FILE_IMPORT_MIN_BYTES: never read whole.
|
|
15
|
+
* One streaming pass counts, hashes and collects its columns, the upload streams
|
|
16
|
+
* from disk, and the import always runs in the background. There is no inline
|
|
17
|
+
* fallback: a file this large cannot travel in one API request.
|
|
18
|
+
*/
|
|
19
|
+
export declare function importLargeFileStreamed(table: string | undefined, options: TablesFileImportOptions, format: RowsFileFormat, fileBytes: number): Promise<Record<string, unknown>>;
|