@oxygen-agent/cli 1.1014.985 → 1.1014.1087
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/command-families/program.d.ts +2 -0
- package/dist/command-families/program.js +241 -0
- package/dist/command-families/tables/automation.d.ts +22 -0
- package/dist/command-families/tables/automation.js +250 -0
- package/dist/command-families/tables/bundle-export.d.ts +7 -0
- package/dist/command-families/tables/bundle-export.js +198 -0
- package/dist/command-families/tables/bundle-format.d.ts +66 -0
- package/dist/command-families/tables/bundle-format.js +274 -0
- package/dist/command-families/tables/bundle-import.d.ts +20 -0
- package/dist/command-families/tables/bundle-import.js +260 -0
- package/dist/command-families/tables/display.d.ts +107 -0
- package/dist/command-families/tables/display.js +229 -0
- package/dist/command-families/tables/import-batches.d.ts +26 -0
- package/dist/command-families/tables/import-batches.js +94 -0
- package/dist/command-families/tables/import-columns.d.ts +24 -0
- package/dist/command-families/tables/import-columns.js +131 -0
- package/dist/command-families/tables/import-upload.d.ts +30 -0
- package/dist/command-families/tables/import-upload.js +203 -0
- package/dist/command-families/tables/import-values.d.ts +10 -0
- package/dist/command-families/tables/import-values.js +60 -0
- package/dist/command-families/tables/imports.d.ts +19 -0
- package/dist/command-families/tables/imports.js +200 -0
- package/dist/command-families/tables/options.d.ts +314 -0
- package/dist/command-families/tables/options.js +1 -0
- package/dist/command-families/tables/register-automation.d.ts +2 -0
- package/dist/command-families/tables/register-automation.js +127 -0
- package/dist/command-families/tables/register-external-imports.d.ts +2 -0
- package/dist/command-families/tables/register-external-imports.js +293 -0
- package/dist/command-families/tables/register-files.d.ts +2 -0
- package/dist/command-families/tables/register-files.js +72 -0
- package/dist/command-families/tables/register-followers.d.ts +2 -0
- package/dist/command-families/tables/register-followers.js +73 -0
- package/dist/command-families/tables/register-lifecycle.d.ts +2 -0
- package/dist/command-families/tables/register-lifecycle.js +114 -0
- package/dist/command-families/tables/register-reads.d.ts +2 -0
- package/dist/command-families/tables/register-reads.js +187 -0
- package/dist/command-families/tables/register-relations.d.ts +2 -0
- package/dist/command-families/tables/register-relations.js +244 -0
- package/dist/command-families/tables/register-row-actions.d.ts +2 -0
- package/dist/command-families/tables/register-row-actions.js +53 -0
- package/dist/command-families/tables/register-row-writes.d.ts +2 -0
- package/dist/command-families/tables/register-row-writes.js +140 -0
- package/dist/command-families/tables/register-views.d.ts +2 -0
- package/dist/command-families/tables/register-views.js +83 -0
- package/dist/command-families/tables/register-watchers.d.ts +2 -0
- package/dist/command-families/tables/register-watchers.js +65 -0
- package/dist/command-families/tables/register-webhooks.d.ts +2 -0
- package/dist/command-families/tables/register-webhooks.js +126 -0
- package/dist/command-families/tables/registration-context.d.ts +8 -0
- package/dist/command-families/tables/registration-context.js +1 -0
- package/dist/command-families/tables/registration.d.ts +3 -0
- package/dist/command-families/tables/registration.js +30 -0
- package/dist/command-families/tables/row-selection.d.ts +5 -0
- package/dist/command-families/tables/row-selection.js +82 -0
- package/dist/command-families/tables/send.d.ts +6 -0
- package/dist/command-families/tables/send.js +111 -0
- package/dist/command-manifest.js +1 -0
- package/dist/commands/copilot.js +8 -0
- package/dist/commands/deliverability.js +4 -2
- package/dist/commands/sequences.js +30 -2
- package/dist/commands/tables.d.ts +1 -3
- package/dist/commands/tables.js +1 -3505
- package/dist/credentials.js +1 -1
- package/dist/customer-operation-commands.js +62 -3
- package/dist/help.js +1 -1
- package/dist/index.d.ts +1 -99
- package/dist/index.js +5 -337
- package/dist/knowledge-mirror.js +1 -1
- package/dist/skills.js +4 -2
- package/dist/update.js +2 -1
- package/dist/workspace-placement-commands.d.ts +3 -0
- package/dist/workspace-placement-commands.js +32 -0
- package/node_modules/@oxygen/shared/dist/agent-sessions.d.ts +4 -0
- package/node_modules/@oxygen/shared/dist/agent-sessions.js +11 -3
- package/node_modules/@oxygen/shared/dist/agents-enabled.d.ts +29 -0
- package/node_modules/@oxygen/shared/dist/agents-enabled.js +51 -0
- package/node_modules/@oxygen/shared/dist/capability-discovery.js +39 -2
- package/node_modules/@oxygen/shared/dist/contracts/agent-sessions.js +1 -1
- package/node_modules/@oxygen/shared/dist/contracts/customer-operations.js +1 -1
- package/node_modules/@oxygen/shared/dist/contracts/guided-activation.js +5 -5
- package/node_modules/@oxygen/shared/dist/contracts/guided-company-files.d.ts +30 -0
- package/node_modules/@oxygen/shared/dist/contracts/guided-company-files.js +53 -0
- package/node_modules/@oxygen/shared/dist/contracts/guided-first-sourcing.d.ts +120 -0
- package/node_modules/@oxygen/shared/dist/contracts/guided-first-sourcing.js +48 -0
- package/node_modules/@oxygen/shared/dist/contracts/guided-onboarding-journey.d.ts +267 -0
- package/node_modules/@oxygen/shared/dist/contracts/guided-onboarding-journey.js +63 -0
- package/node_modules/@oxygen/shared/dist/contracts/operations.d.ts +6 -0
- package/node_modules/@oxygen/shared/dist/contracts/operations.js +9 -4
- package/node_modules/@oxygen/shared/dist/contracts/table-query-raw.d.ts +2 -0
- package/node_modules/@oxygen/shared/dist/contracts/table-query-raw.js +3 -0
- package/node_modules/@oxygen/shared/dist/contracts/teammates.js +17 -15
- package/node_modules/@oxygen/shared/dist/email-tracking-token.d.ts +2 -0
- package/node_modules/@oxygen/shared/dist/email-tracking-token.js +1 -0
- package/node_modules/@oxygen/shared/dist/feature-gates.d.ts +4 -0
- package/node_modules/@oxygen/shared/dist/feature-gates.js +5 -0
- package/node_modules/@oxygen/shared/dist/file-import.d.ts +46 -0
- package/node_modules/@oxygen/shared/dist/file-import.js +138 -0
- package/node_modules/@oxygen/shared/dist/guided-activation.d.ts +8 -0
- package/node_modules/@oxygen/shared/dist/guided-company-files.d.ts +13 -0
- package/node_modules/@oxygen/shared/dist/guided-company-files.js +14 -0
- package/node_modules/@oxygen/shared/dist/guided-first-sourcing.d.ts +116 -0
- package/node_modules/@oxygen/shared/dist/guided-first-sourcing.js +11 -0
- package/node_modules/@oxygen/shared/dist/guided-onboarding-answers.d.ts +4 -0
- package/node_modules/@oxygen/shared/dist/guided-onboarding-answers.js +42 -0
- package/node_modules/@oxygen/shared/dist/guided-onboarding-company-context.d.ts +22 -0
- package/node_modules/@oxygen/shared/dist/guided-onboarding-company-context.js +38 -0
- package/node_modules/@oxygen/shared/dist/guided-onboarding-journey.d.ts +147 -0
- package/node_modules/@oxygen/shared/dist/guided-onboarding-journey.fixture.d.ts +12 -0
- package/node_modules/@oxygen/shared/dist/guided-onboarding-journey.fixture.js +83 -0
- package/node_modules/@oxygen/shared/dist/guided-onboarding-journey.js +44 -0
- package/node_modules/@oxygen/shared/dist/guided-onboarding-prospects.d.ts +7 -0
- package/node_modules/@oxygen/shared/dist/guided-onboarding-prospects.js +13 -0
- package/node_modules/@oxygen/shared/dist/guided-onboarding-signals.d.ts +4 -0
- package/node_modules/@oxygen/shared/dist/guided-onboarding-signals.js +34 -0
- package/node_modules/@oxygen/shared/dist/guided-trial-billing.d.ts +26 -0
- package/node_modules/@oxygen/shared/dist/guided-trial-billing.js +19 -0
- package/node_modules/@oxygen/shared/dist/index.d.ts +0 -1
- package/node_modules/@oxygen/shared/dist/index.js +0 -6
- package/node_modules/@oxygen/shared/dist/table-limits.d.ts +1 -0
- package/node_modules/@oxygen/shared/dist/table-limits.js +6 -0
- package/node_modules/@oxygen/shared/dist/teammates.d.ts +13 -8
- package/node_modules/@oxygen/shared/dist/teammates.js +20 -11
- package/node_modules/@oxygen/shared/dist/user-capability-routing.js +78 -0
- package/node_modules/@oxygen/shared/dist/version.generated.d.ts +1 -1
- package/node_modules/@oxygen/shared/dist/version.generated.js +1 -1
- package/node_modules/@oxygen/shared/dist/workspace-placement-view.d.ts +149 -0
- package/node_modules/@oxygen/shared/dist/workspace-placement-view.js +101 -0
- package/node_modules/@oxygen/shared/dist/workspace-placement.d.ts +77 -0
- package/node_modules/@oxygen/shared/dist/workspace-placement.js +94 -0
- package/package.json +1 -1
|
@@ -0,0 +1,198 @@
|
|
|
1
|
+
import { readPositiveInt, readRecordString } from "../../cli-values.js";
|
|
2
|
+
import { normalizeExportRowsFormat, formatRows } from "../../command-support.js";
|
|
3
|
+
import { requestOxygen } from "../../http-client.js";
|
|
4
|
+
import { OxygenError } from "@oxygen/shared/cli-result";
|
|
5
|
+
import { writeFileSync, createWriteStream, renameSync, unlinkSync } from "node:fs";
|
|
6
|
+
import { resolve } from "node:path";
|
|
7
|
+
import { TABLE_BUNDLE_DEFAULT_PAGE_SIZE, TABLE_BUNDLE_LEGACY_SCHEMA_VERSION, TABLE_BUNDLE_MAX_PAGE_SIZE, TABLE_BUNDLE_SCHEMA_VERSION, readBundleNumber } from "./bundle-format.js";
|
|
8
|
+
import { toBundleColumn } from "./bundle-import.js";
|
|
9
|
+
import { tableWebUrl } from "./import-values.js";
|
|
10
|
+
export async function exportRows(table, options) {
|
|
11
|
+
const format = normalizeExportRowsFormat(options.format);
|
|
12
|
+
const limit = readPositiveInt(options.limit);
|
|
13
|
+
const result = await requestOxygen("/api/cli/tables/query", {
|
|
14
|
+
method: "POST",
|
|
15
|
+
body: {
|
|
16
|
+
table,
|
|
17
|
+
...(limit ? { limit } : {}),
|
|
18
|
+
},
|
|
19
|
+
});
|
|
20
|
+
const formatted = formatRows(result.rows, format, result.columns);
|
|
21
|
+
if (options.output) {
|
|
22
|
+
writeFileSync(options.output, formatted.content);
|
|
23
|
+
}
|
|
24
|
+
return {
|
|
25
|
+
table: result.table ?? null,
|
|
26
|
+
format,
|
|
27
|
+
rowCount: result.rows.length,
|
|
28
|
+
output: options.output ?? null,
|
|
29
|
+
...(options.output ? {} : { content: formatted.content }),
|
|
30
|
+
...(formatted.rescuedCount > 0 ? { rescuedNumericCells: formatted.rescuedCount } : {}),
|
|
31
|
+
};
|
|
32
|
+
}
|
|
33
|
+
// skipcq: JS-R1005 — intentional branching across describe/query pagination, column-definition normalization, and output sink (stdout/file)
|
|
34
|
+
export async function exportTableBundle(table, options) {
|
|
35
|
+
const pageSize = Math.min(readPositiveInt(options.pageSize) ?? TABLE_BUNDLE_DEFAULT_PAGE_SIZE, TABLE_BUNDLE_MAX_PAGE_SIZE);
|
|
36
|
+
// Pull the canonical schema (incl. enrichment/tool definitions and
|
|
37
|
+
// semantic types) from `describe`; the paginated query response only
|
|
38
|
+
// surfaces the user-visible projection.
|
|
39
|
+
const describe = await requestOxygen("/api/cli/tables/describe", { method: "POST", body: { table } });
|
|
40
|
+
const tableMeta = describe.table ?? null;
|
|
41
|
+
const columns = (describe.columns ?? []).map(toBundleColumn);
|
|
42
|
+
const tableId = readRecordString(tableMeta, "id");
|
|
43
|
+
const tableSlug = readRecordString(tableMeta, "slug");
|
|
44
|
+
const projectSlug = readRecordString(tableMeta, "projectSlug");
|
|
45
|
+
const tableSummary = {
|
|
46
|
+
id: tableId ?? null,
|
|
47
|
+
slug: tableSlug ?? null,
|
|
48
|
+
name: readRecordString(tableMeta, "displayName") ?? readRecordString(tableMeta, "name") ?? null,
|
|
49
|
+
projectSlug: projectSlug ?? null,
|
|
50
|
+
};
|
|
51
|
+
const exportedAt = new Date().toISOString();
|
|
52
|
+
// With --output the bundle streams to disk one row per line, so memory stays
|
|
53
|
+
// flat however many rows the table has. The file is written under a .partial
|
|
54
|
+
// name and renamed only after the totals trailer lands, so an interrupted
|
|
55
|
+
// export can never be mistaken for a complete bundle.
|
|
56
|
+
const outputPath = options.output ? resolve(options.output) : null;
|
|
57
|
+
const partialPath = outputPath ? `${outputPath}.partial` : null;
|
|
58
|
+
const stream = partialPath ? createWriteStream(partialPath, { encoding: "utf8" }) : null;
|
|
59
|
+
const rows = [];
|
|
60
|
+
let cursor = null;
|
|
61
|
+
let expectedTotal = null;
|
|
62
|
+
let pageCount = 0;
|
|
63
|
+
let rowCount = 0;
|
|
64
|
+
let hasMoreFlag = false;
|
|
65
|
+
try {
|
|
66
|
+
if (stream) {
|
|
67
|
+
await writeBundleLine(stream, JSON.stringify({
|
|
68
|
+
schemaVersion: TABLE_BUNDLE_SCHEMA_VERSION,
|
|
69
|
+
exportedAt,
|
|
70
|
+
table: tableSummary,
|
|
71
|
+
columns,
|
|
72
|
+
}));
|
|
73
|
+
}
|
|
74
|
+
do {
|
|
75
|
+
const requestBody = { table, limit: pageSize };
|
|
76
|
+
if (cursor)
|
|
77
|
+
requestBody.cursor = cursor;
|
|
78
|
+
const page = await requestOxygen("/api/cli/tables/query", { method: "POST", body: requestBody });
|
|
79
|
+
pageCount += 1;
|
|
80
|
+
if (expectedTotal === null) {
|
|
81
|
+
expectedTotal = readBundleNumber(page.totalCount) ?? readBundleNumber(page.total_count) ?? null;
|
|
82
|
+
}
|
|
83
|
+
for (const row of page.rows ?? []) {
|
|
84
|
+
rowCount += 1;
|
|
85
|
+
if (stream)
|
|
86
|
+
await writeBundleLine(stream, JSON.stringify({ row }));
|
|
87
|
+
else
|
|
88
|
+
rows.push(row);
|
|
89
|
+
}
|
|
90
|
+
cursor = typeof page.nextCursor === "string" && page.nextCursor.length > 0 ? page.nextCursor : null;
|
|
91
|
+
hasMoreFlag = Boolean(page.hasMore);
|
|
92
|
+
// Defensive: hasMore=false should always mean cursor=null. If a future
|
|
93
|
+
// server change ever sets one without the other, stop iterating on
|
|
94
|
+
// hasMore=false so we don't loop forever.
|
|
95
|
+
if (!hasMoreFlag)
|
|
96
|
+
cursor = null;
|
|
97
|
+
} while (cursor);
|
|
98
|
+
if (expectedTotal !== null && rowCount !== expectedTotal) {
|
|
99
|
+
throw new OxygenError("bundle_export_incomplete", "Cursor pagination ended before every row was written. The bundle would be a silent under-export.", {
|
|
100
|
+
details: {
|
|
101
|
+
table,
|
|
102
|
+
totalCount: expectedTotal,
|
|
103
|
+
exportedCount: rowCount,
|
|
104
|
+
missing: expectedTotal - rowCount,
|
|
105
|
+
pages: pageCount,
|
|
106
|
+
},
|
|
107
|
+
exitCode: 1,
|
|
108
|
+
});
|
|
109
|
+
}
|
|
110
|
+
const totals = {
|
|
111
|
+
rowCount,
|
|
112
|
+
...(expectedTotal !== null ? { sourceTotalCount: expectedTotal } : {}),
|
|
113
|
+
pages: pageCount,
|
|
114
|
+
pageSize,
|
|
115
|
+
};
|
|
116
|
+
if (stream && partialPath && outputPath) {
|
|
117
|
+
await writeBundleLine(stream, JSON.stringify({ totals }));
|
|
118
|
+
await finishBundleStream(stream);
|
|
119
|
+
renameSync(partialPath, outputPath);
|
|
120
|
+
}
|
|
121
|
+
const summary = {
|
|
122
|
+
table_id: tableId ?? null,
|
|
123
|
+
table_slug: tableSlug ?? null,
|
|
124
|
+
schema_version: stream ? TABLE_BUNDLE_SCHEMA_VERSION : TABLE_BUNDLE_LEGACY_SCHEMA_VERSION,
|
|
125
|
+
format: stream ? "jsonl" : "json",
|
|
126
|
+
column_count: columns.length,
|
|
127
|
+
row_count: rowCount,
|
|
128
|
+
pages: pageCount,
|
|
129
|
+
page_size: pageSize,
|
|
130
|
+
output: options.output ?? null,
|
|
131
|
+
...(tableId ? { web_url: tableWebUrl(tableId) } : tableSlug ? { web_url: tableWebUrl(tableSlug) } : {}),
|
|
132
|
+
};
|
|
133
|
+
// With --output, the file is the source of truth; keep the stdout response
|
|
134
|
+
// a small summary so it's readable.
|
|
135
|
+
if (stream)
|
|
136
|
+
return summary;
|
|
137
|
+
// When piping to stdout, emit the full single-document bundle so it can be
|
|
138
|
+
// redirected into a file. That document has to fit in one string, so say
|
|
139
|
+
// so — with the fix — instead of dying on a bare RangeError.
|
|
140
|
+
const bundle = {
|
|
141
|
+
schemaVersion: TABLE_BUNDLE_LEGACY_SCHEMA_VERSION,
|
|
142
|
+
exportedAt,
|
|
143
|
+
table: tableSummary,
|
|
144
|
+
columns,
|
|
145
|
+
rows,
|
|
146
|
+
totals,
|
|
147
|
+
};
|
|
148
|
+
assertBundleSerializable(bundle, rowCount);
|
|
149
|
+
return { ...summary, bundle };
|
|
150
|
+
}
|
|
151
|
+
catch (error) {
|
|
152
|
+
if (stream) {
|
|
153
|
+
stream.destroy();
|
|
154
|
+
if (partialPath) {
|
|
155
|
+
try {
|
|
156
|
+
unlinkSync(partialPath);
|
|
157
|
+
}
|
|
158
|
+
catch {
|
|
159
|
+
// The partial file may never have been created; nothing to clean.
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
throw error;
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
// Writes one bundle line and honors the stream's backpressure, so a 50k-row
|
|
167
|
+
// export never accumulates the file in memory while the disk catches up.
|
|
168
|
+
export function writeBundleLine(stream, line) {
|
|
169
|
+
return new Promise((resolveLine, rejectLine) => {
|
|
170
|
+
const onError = (error) => rejectLine(error);
|
|
171
|
+
if (stream.write(`${line}\n`, "utf8")) {
|
|
172
|
+
resolveLine();
|
|
173
|
+
return;
|
|
174
|
+
}
|
|
175
|
+
stream.once("error", onError);
|
|
176
|
+
stream.once("drain", () => {
|
|
177
|
+
stream.off("error", onError);
|
|
178
|
+
resolveLine();
|
|
179
|
+
});
|
|
180
|
+
});
|
|
181
|
+
}
|
|
182
|
+
export function finishBundleStream(stream) {
|
|
183
|
+
return new Promise((resolveStream, rejectStream) => {
|
|
184
|
+
stream.once("error", rejectStream);
|
|
185
|
+
stream.end(() => resolveStream());
|
|
186
|
+
});
|
|
187
|
+
}
|
|
188
|
+
export function assertBundleSerializable(bundle, rowCount) {
|
|
189
|
+
try {
|
|
190
|
+
JSON.stringify(bundle);
|
|
191
|
+
}
|
|
192
|
+
catch (error) {
|
|
193
|
+
if (error instanceof RangeError) {
|
|
194
|
+
throw new OxygenError("bundle_too_large", `This table's ${rowCount} rows do not fit in one JSON document. Re-run with --output <path>: the bundle then streams to the file one row per line, with no size limit.`, { details: { row_count: rowCount }, exitCode: 1 });
|
|
195
|
+
}
|
|
196
|
+
throw error;
|
|
197
|
+
}
|
|
198
|
+
}
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
import { QueryColumnDescriptor } from "../../command-support.js";
|
|
2
|
+
import type { RowsFileFormat } from "@oxygen/shared/file-import";
|
|
3
|
+
export declare const TABLE_BUNDLE_SCHEMA_VERSION = 2;
|
|
4
|
+
export declare const TABLE_BUNDLE_LEGACY_SCHEMA_VERSION = 1;
|
|
5
|
+
export declare const TABLE_BUNDLE_SUPPORTED_SCHEMA_VERSIONS: number[];
|
|
6
|
+
export declare const TABLE_BUNDLE_MAX_PAGE_SIZE = 1000;
|
|
7
|
+
export declare const TABLE_BUNDLE_DEFAULT_PAGE_SIZE = 500;
|
|
8
|
+
export declare const TABLE_BUNDLE_HEADER_SCAN_BYTES: number;
|
|
9
|
+
export declare const TABLE_BUNDLE_TRAILER_SCAN_BYTES: number;
|
|
10
|
+
export type QueryPageResponse = {
|
|
11
|
+
table?: {
|
|
12
|
+
id?: string;
|
|
13
|
+
slug?: string;
|
|
14
|
+
displayName?: string;
|
|
15
|
+
name?: string;
|
|
16
|
+
};
|
|
17
|
+
columns?: QueryColumnDescriptor[];
|
|
18
|
+
rows: Record<string, unknown>[];
|
|
19
|
+
nextCursor?: string | null;
|
|
20
|
+
hasMore?: boolean;
|
|
21
|
+
totalCount?: number;
|
|
22
|
+
total_count?: number;
|
|
23
|
+
returnedCount?: number;
|
|
24
|
+
returned_count?: number;
|
|
25
|
+
};
|
|
26
|
+
export type ParsedTableBundle = {
|
|
27
|
+
schemaVersion: number;
|
|
28
|
+
tableName: string | null;
|
|
29
|
+
tableSummary: Record<string, unknown> | null;
|
|
30
|
+
columns: Record<string, unknown>[];
|
|
31
|
+
rows: Record<string, unknown>[];
|
|
32
|
+
};
|
|
33
|
+
export type OpenedTableBundle = {
|
|
34
|
+
schemaVersion: number;
|
|
35
|
+
tableName: string | null;
|
|
36
|
+
tableSummary: Record<string, unknown> | null;
|
|
37
|
+
columns: Record<string, unknown>[];
|
|
38
|
+
/**
|
|
39
|
+
* Rows the bundle states it holds: the array length of a single document,
|
|
40
|
+
* the totals trailer of a streamed bundle.
|
|
41
|
+
*/
|
|
42
|
+
rowCount: number;
|
|
43
|
+
rows: () => AsyncIterable<Record<string, unknown>>;
|
|
44
|
+
/** Single-document bundles only: every row is already in memory. */
|
|
45
|
+
preloadedRows: Record<string, unknown>[] | null;
|
|
46
|
+
};
|
|
47
|
+
export declare function openTableBundle(filePath: string): Promise<OpenedTableBundle>;
|
|
48
|
+
export declare function iterateArrayRows(rows: Record<string, unknown>[]): AsyncGenerator<Record<string, unknown>>;
|
|
49
|
+
export declare function openStreamedTableBundle(filePath: string, header: Record<string, unknown>): OpenedTableBundle;
|
|
50
|
+
export declare function iterateStreamedBundleRows(filePath: string): AsyncGenerator<Record<string, unknown>>;
|
|
51
|
+
export declare function readBundleBatches(input: {
|
|
52
|
+
bundle: OpenedTableBundle;
|
|
53
|
+
validColumnKeys: Set<string>;
|
|
54
|
+
upsertKey: string | null;
|
|
55
|
+
batchSize: number;
|
|
56
|
+
tableId: string;
|
|
57
|
+
resumeCommand: string;
|
|
58
|
+
}): AsyncGenerator<Record<string, unknown>[]>;
|
|
59
|
+
export declare function readFirstBundleLine(filePath: string): string | null;
|
|
60
|
+
export declare function readLastBundleLine(filePath: string): string;
|
|
61
|
+
export declare function parseJsonObjectOrNull(text: string): Record<string, unknown> | null;
|
|
62
|
+
export declare function parseBundleFile(text: string): ParsedTableBundle;
|
|
63
|
+
export declare function stripRowForImport(row: Record<string, unknown>, validColumnKeys: Set<string>): Record<string, unknown>;
|
|
64
|
+
export declare function readBundleNumber(value: unknown): number | null;
|
|
65
|
+
export declare function readRowsFile(path: string, format: RowsFileFormat, sheet?: string): Promise<Record<string, unknown>[]>;
|
|
66
|
+
export declare function normalizeCreatedTableUpsertKey(value: string | undefined, keyBySource: Record<string, string>): string | undefined;
|
|
@@ -0,0 +1,274 @@
|
|
|
1
|
+
import { readRecordString } from "../../cli-values.js";
|
|
2
|
+
import { readRecord } from "../../command-support.js";
|
|
3
|
+
import { OxygenError } from "@oxygen/shared/cli-result";
|
|
4
|
+
import { parseRowsFileBuffer } from "@oxygen/shared/file-import";
|
|
5
|
+
import { existsSync, readFileSync, createReadStream, statSync, openSync, readSync, closeSync } from "node:fs";
|
|
6
|
+
import { createInterface } from "node:readline/promises";
|
|
7
|
+
// Bundle formats. Version 2 is line-delimited: a header line carrying the
|
|
8
|
+
// schema, then one `{"row":…}` line per row, then a `{"totals":…}` trailer.
|
|
9
|
+
// It exists because the version-1 single-document bundle is assembled in memory
|
|
10
|
+
// and serialized once, and a 52k-row table carrying LinkedIn profile text pushed
|
|
11
|
+
// that one string past V8's limit ("Invalid string length") on 2026-09-14: the
|
|
12
|
+
// export died after twelve minutes with nothing on disk, and a 96-column table
|
|
13
|
+
// on the other side of the same copy could never have been read back into
|
|
14
|
+
// memory by the importer either. `--output` now streams version 2; import reads
|
|
15
|
+
// both versions, streaming version 2 so file size no longer matters.
|
|
16
|
+
export const TABLE_BUNDLE_SCHEMA_VERSION = 2;
|
|
17
|
+
export const TABLE_BUNDLE_LEGACY_SCHEMA_VERSION = 1;
|
|
18
|
+
export const TABLE_BUNDLE_SUPPORTED_SCHEMA_VERSIONS = [TABLE_BUNDLE_LEGACY_SCHEMA_VERSION, TABLE_BUNDLE_SCHEMA_VERSION];
|
|
19
|
+
export const TABLE_BUNDLE_MAX_PAGE_SIZE = 1000;
|
|
20
|
+
export const TABLE_BUNDLE_DEFAULT_PAGE_SIZE = 500;
|
|
21
|
+
// How much of a bundle file is read to find its header line and totals trailer
|
|
22
|
+
// without loading the whole file: 8 MiB comfortably holds a header for a
|
|
23
|
+
// 100-column table with long AI prompts, and the trailer is one short line.
|
|
24
|
+
export const TABLE_BUNDLE_HEADER_SCAN_BYTES = 8 * 1024 * 1024;
|
|
25
|
+
export const TABLE_BUNDLE_TRAILER_SCAN_BYTES = 1024 * 1024;
|
|
26
|
+
// Opens either bundle format. A streamed (version 2) bundle is recognised by
|
|
27
|
+
// its header line and read line by line, so a multi-GB file never has to fit
|
|
28
|
+
// in memory; a single-document (version 1) bundle is parsed whole, as before.
|
|
29
|
+
export async function openTableBundle(filePath) {
|
|
30
|
+
if (!existsSync(filePath)) {
|
|
31
|
+
throw new OxygenError("invalid_input", `Bundle file not found: ${filePath}`, {
|
|
32
|
+
details: { file: filePath },
|
|
33
|
+
exitCode: 1,
|
|
34
|
+
});
|
|
35
|
+
}
|
|
36
|
+
const firstLine = readFirstBundleLine(filePath);
|
|
37
|
+
const header = firstLine === null ? null : parseJsonObjectOrNull(firstLine);
|
|
38
|
+
if (header && readBundleNumber(header.schemaVersion) === TABLE_BUNDLE_SCHEMA_VERSION) {
|
|
39
|
+
return openStreamedTableBundle(filePath, header);
|
|
40
|
+
}
|
|
41
|
+
let raw;
|
|
42
|
+
try {
|
|
43
|
+
raw = readFileSync(filePath, "utf8");
|
|
44
|
+
}
|
|
45
|
+
catch (error) {
|
|
46
|
+
if (error instanceof RangeError || error.code === "ERR_STRING_TOO_LONG") {
|
|
47
|
+
throw new OxygenError("bundle_too_large", "This bundle is one JSON document too large to load into memory. Re-export it with the current CLI (`oxygen tables export-bundle <table> --output <path>`), which writes one row per line and imports at any size.", { details: { file: filePath }, exitCode: 1 });
|
|
48
|
+
}
|
|
49
|
+
throw error;
|
|
50
|
+
}
|
|
51
|
+
const parsed = parseBundleFile(raw);
|
|
52
|
+
return {
|
|
53
|
+
schemaVersion: parsed.schemaVersion,
|
|
54
|
+
tableName: parsed.tableName,
|
|
55
|
+
tableSummary: parsed.tableSummary,
|
|
56
|
+
columns: parsed.columns,
|
|
57
|
+
rowCount: parsed.rows.length,
|
|
58
|
+
preloadedRows: parsed.rows,
|
|
59
|
+
rows: () => iterateArrayRows(parsed.rows),
|
|
60
|
+
};
|
|
61
|
+
}
|
|
62
|
+
export async function* iterateArrayRows(rows) {
|
|
63
|
+
for (const row of rows)
|
|
64
|
+
yield row;
|
|
65
|
+
}
|
|
66
|
+
export function openStreamedTableBundle(filePath, header) {
|
|
67
|
+
const rawColumns = header.columns;
|
|
68
|
+
if (!Array.isArray(rawColumns)) {
|
|
69
|
+
throw new OxygenError("invalid_bundle", "Streamed bundle header is missing a columns array.", {
|
|
70
|
+
details: { file: filePath },
|
|
71
|
+
exitCode: 1,
|
|
72
|
+
});
|
|
73
|
+
}
|
|
74
|
+
const tableSummary = readRecord(header, "table");
|
|
75
|
+
// The totals trailer is the last thing the exporter writes, so its absence
|
|
76
|
+
// means the export was interrupted: refuse before creating anything rather
|
|
77
|
+
// than importing a silently truncated table.
|
|
78
|
+
const trailer = parseJsonObjectOrNull(readLastBundleLine(filePath));
|
|
79
|
+
const totals = trailer ? readRecord(trailer, "totals") : null;
|
|
80
|
+
const rowCount = totals ? readBundleNumber(totals.rowCount) : null;
|
|
81
|
+
if (rowCount === null) {
|
|
82
|
+
throw new OxygenError("invalid_bundle", "This streamed bundle has no totals trailer, so the export that produced it was interrupted before it finished. Re-run `oxygen tables export-bundle` and import the complete file.", { details: { file: filePath }, exitCode: 1 });
|
|
83
|
+
}
|
|
84
|
+
return {
|
|
85
|
+
schemaVersion: TABLE_BUNDLE_SCHEMA_VERSION,
|
|
86
|
+
tableName: tableSummary
|
|
87
|
+
? (readRecordString(tableSummary, "name") ?? readRecordString(tableSummary, "displayName"))
|
|
88
|
+
: null,
|
|
89
|
+
tableSummary,
|
|
90
|
+
columns: rawColumns.filter((entry) => Boolean(entry) && typeof entry === "object" && !Array.isArray(entry)),
|
|
91
|
+
rowCount,
|
|
92
|
+
preloadedRows: null,
|
|
93
|
+
rows: () => iterateStreamedBundleRows(filePath),
|
|
94
|
+
};
|
|
95
|
+
}
|
|
96
|
+
export async function* iterateStreamedBundleRows(filePath) {
|
|
97
|
+
const fileStream = createReadStream(filePath, { encoding: "utf8" });
|
|
98
|
+
const reader = createInterface({ input: fileStream, crlfDelay: Infinity });
|
|
99
|
+
let lineNumber = 0;
|
|
100
|
+
try {
|
|
101
|
+
for await (const line of reader) {
|
|
102
|
+
lineNumber += 1;
|
|
103
|
+
if (lineNumber === 1 || !line.trim())
|
|
104
|
+
continue;
|
|
105
|
+
const entry = parseJsonObjectOrNull(line);
|
|
106
|
+
if (!entry) {
|
|
107
|
+
throw new OxygenError("invalid_bundle", `Line ${lineNumber} of the bundle is not a JSON object.`, {
|
|
108
|
+
details: { file: filePath, line: lineNumber },
|
|
109
|
+
exitCode: 1,
|
|
110
|
+
});
|
|
111
|
+
}
|
|
112
|
+
if ("totals" in entry)
|
|
113
|
+
return;
|
|
114
|
+
const row = readRecord(entry, "row");
|
|
115
|
+
if (!row) {
|
|
116
|
+
throw new OxygenError("invalid_bundle", `Line ${lineNumber} of the bundle is neither a row nor the totals trailer.`, { details: { file: filePath, line: lineNumber }, exitCode: 1 });
|
|
117
|
+
}
|
|
118
|
+
yield row;
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
finally {
|
|
122
|
+
reader.close();
|
|
123
|
+
fileStream.destroy();
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
// Batches the bundle's rows for the shared import writer, stripping tenant-local
|
|
127
|
+
// fields and enforcing the upsert key on the way through.
|
|
128
|
+
export async function* readBundleBatches(input) {
|
|
129
|
+
let batch = [];
|
|
130
|
+
let rowNumber = 0;
|
|
131
|
+
for await (const rawRow of input.bundle.rows()) {
|
|
132
|
+
rowNumber += 1;
|
|
133
|
+
const row = stripRowForImport(rawRow, input.validColumnKeys);
|
|
134
|
+
if (input.upsertKey && (row[input.upsertKey] === undefined || row[input.upsertKey] === null)) {
|
|
135
|
+
throw new OxygenError("invalid_bundle", `Row ${rowNumber} is missing a value for the upsert key "${input.upsertKey}".`, {
|
|
136
|
+
details: {
|
|
137
|
+
key: input.upsertKey,
|
|
138
|
+
row_number: rowNumber,
|
|
139
|
+
table_id: input.tableId,
|
|
140
|
+
resume_command: input.resumeCommand,
|
|
141
|
+
},
|
|
142
|
+
exitCode: 1,
|
|
143
|
+
});
|
|
144
|
+
}
|
|
145
|
+
batch.push(row);
|
|
146
|
+
if (batch.length >= input.batchSize) {
|
|
147
|
+
yield batch;
|
|
148
|
+
batch = [];
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
if (batch.length > 0)
|
|
152
|
+
yield batch;
|
|
153
|
+
}
|
|
154
|
+
// Reads up to the first newline without loading the file. Returns null when no
|
|
155
|
+
// newline appears inside the scan window and the file is larger than it, which
|
|
156
|
+
// is what a huge single-document bundle looks like.
|
|
157
|
+
export function readFirstBundleLine(filePath) {
|
|
158
|
+
const size = statSync(filePath).size;
|
|
159
|
+
const span = Math.min(size, TABLE_BUNDLE_HEADER_SCAN_BYTES);
|
|
160
|
+
if (span === 0)
|
|
161
|
+
return "";
|
|
162
|
+
const fd = openSync(filePath, "r");
|
|
163
|
+
try {
|
|
164
|
+
const buffer = Buffer.alloc(span);
|
|
165
|
+
const read = readSync(fd, buffer, 0, span, 0);
|
|
166
|
+
const text = buffer.subarray(0, read).toString("utf8");
|
|
167
|
+
const newline = text.indexOf("\n");
|
|
168
|
+
if (newline >= 0)
|
|
169
|
+
return text.slice(0, newline);
|
|
170
|
+
return read < size ? null : text;
|
|
171
|
+
}
|
|
172
|
+
finally {
|
|
173
|
+
closeSync(fd);
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
export function readLastBundleLine(filePath) {
|
|
177
|
+
const size = statSync(filePath).size;
|
|
178
|
+
const span = Math.min(size, TABLE_BUNDLE_TRAILER_SCAN_BYTES);
|
|
179
|
+
if (span === 0)
|
|
180
|
+
return "";
|
|
181
|
+
const fd = openSync(filePath, "r");
|
|
182
|
+
try {
|
|
183
|
+
const buffer = Buffer.alloc(span);
|
|
184
|
+
const read = readSync(fd, buffer, 0, span, size - span);
|
|
185
|
+
const lines = buffer.subarray(0, read).toString("utf8").split("\n").map((line) => line.trim()).filter(Boolean);
|
|
186
|
+
return lines.at(-1) ?? "";
|
|
187
|
+
}
|
|
188
|
+
finally {
|
|
189
|
+
closeSync(fd);
|
|
190
|
+
}
|
|
191
|
+
}
|
|
192
|
+
export function parseJsonObjectOrNull(text) {
|
|
193
|
+
try {
|
|
194
|
+
const parsed = JSON.parse(text);
|
|
195
|
+
return parsed && typeof parsed === "object" && !Array.isArray(parsed)
|
|
196
|
+
? parsed
|
|
197
|
+
: null;
|
|
198
|
+
}
|
|
199
|
+
catch {
|
|
200
|
+
return null;
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
export function parseBundleFile(text) {
|
|
204
|
+
let parsed;
|
|
205
|
+
try {
|
|
206
|
+
parsed = JSON.parse(text);
|
|
207
|
+
}
|
|
208
|
+
catch (error) {
|
|
209
|
+
throw new OxygenError("invalid_bundle", "Bundle file is not valid JSON.", {
|
|
210
|
+
details: { reason: error instanceof Error ? error.message : "unknown" },
|
|
211
|
+
exitCode: 1,
|
|
212
|
+
});
|
|
213
|
+
}
|
|
214
|
+
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
|
|
215
|
+
throw new OxygenError("invalid_bundle", "Bundle file must be a JSON object.", { exitCode: 1 });
|
|
216
|
+
}
|
|
217
|
+
const record = parsed;
|
|
218
|
+
const schemaVersion = typeof record.schemaVersion === "number" ? record.schemaVersion : 1;
|
|
219
|
+
// A version-2 bundle never reaches this parser (openTableBundle streams it),
|
|
220
|
+
// so a single document claiming any other version is from a CLI this one
|
|
221
|
+
// does not know.
|
|
222
|
+
if (schemaVersion !== TABLE_BUNDLE_LEGACY_SCHEMA_VERSION) {
|
|
223
|
+
throw new OxygenError("unsupported_bundle_version", `Bundle schema version ${schemaVersion} is not supported by this CLI.`, {
|
|
224
|
+
details: { supported: TABLE_BUNDLE_SUPPORTED_SCHEMA_VERSIONS, got: schemaVersion },
|
|
225
|
+
exitCode: 1,
|
|
226
|
+
});
|
|
227
|
+
}
|
|
228
|
+
const rawColumns = record.columns;
|
|
229
|
+
if (!Array.isArray(rawColumns)) {
|
|
230
|
+
throw new OxygenError("invalid_bundle", "Bundle is missing a columns array.", { exitCode: 1 });
|
|
231
|
+
}
|
|
232
|
+
const rawRows = record.rows;
|
|
233
|
+
if (!Array.isArray(rawRows)) {
|
|
234
|
+
throw new OxygenError("invalid_bundle", "Bundle is missing a rows array.", { exitCode: 1 });
|
|
235
|
+
}
|
|
236
|
+
const tableSummary = record.table && typeof record.table === "object" && !Array.isArray(record.table)
|
|
237
|
+
? record.table
|
|
238
|
+
: null;
|
|
239
|
+
return {
|
|
240
|
+
schemaVersion,
|
|
241
|
+
tableName: tableSummary
|
|
242
|
+
? (readRecordString(tableSummary, "name") ?? readRecordString(tableSummary, "displayName"))
|
|
243
|
+
: null,
|
|
244
|
+
tableSummary,
|
|
245
|
+
columns: rawColumns.filter((entry) => Boolean(entry) && typeof entry === "object" && !Array.isArray(entry)),
|
|
246
|
+
rows: rawRows.filter((entry) => Boolean(entry) && typeof entry === "object" && !Array.isArray(entry)),
|
|
247
|
+
};
|
|
248
|
+
}
|
|
249
|
+
export function stripRowForImport(row, validColumnKeys) {
|
|
250
|
+
const out = {};
|
|
251
|
+
for (const [key, value] of Object.entries(row)) {
|
|
252
|
+
if (key === "_row_id" || key === "_created_at" || key === "_updated_at")
|
|
253
|
+
continue;
|
|
254
|
+
if (validColumnKeys.size > 0 && !validColumnKeys.has(key))
|
|
255
|
+
continue;
|
|
256
|
+
out[key] = value;
|
|
257
|
+
}
|
|
258
|
+
return out;
|
|
259
|
+
}
|
|
260
|
+
export function readBundleNumber(value) {
|
|
261
|
+
if (typeof value === "number" && Number.isFinite(value))
|
|
262
|
+
return value;
|
|
263
|
+
if (typeof value === "string" && value.trim() && Number.isFinite(Number(value)))
|
|
264
|
+
return Number(value);
|
|
265
|
+
return null;
|
|
266
|
+
}
|
|
267
|
+
export async function readRowsFile(path, format, sheet) {
|
|
268
|
+
return await parseRowsFileBuffer(readFileSync(path), format, sheet ? { sheet } : {});
|
|
269
|
+
}
|
|
270
|
+
export function normalizeCreatedTableUpsertKey(value, keyBySource) {
|
|
271
|
+
if (!value)
|
|
272
|
+
return undefined;
|
|
273
|
+
return keyBySource[value] ?? value;
|
|
274
|
+
}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import { QueryColumnDescriptor } from "../../command-support.js";
|
|
2
|
+
import { TablesImportBundleOptions } from "./options.js";
|
|
3
|
+
export declare function importTableBundle(// skipcq: JS-R1005
|
|
4
|
+
options: TablesImportBundleOptions, binaryName: string): Promise<Record<string, unknown>>;
|
|
5
|
+
export declare function buildBundleResumeCommand(input: {
|
|
6
|
+
file: string;
|
|
7
|
+
tableId: string;
|
|
8
|
+
key: string | null;
|
|
9
|
+
batchSize: number;
|
|
10
|
+
binaryName: string;
|
|
11
|
+
}): string;
|
|
12
|
+
export declare function toBundleColumn(column: QueryColumnDescriptor): Record<string, unknown>;
|
|
13
|
+
export declare function bundleColumnToCreateInput(column: Record<string, unknown>): {
|
|
14
|
+
label: string;
|
|
15
|
+
key?: string;
|
|
16
|
+
dataType?: string;
|
|
17
|
+
kind?: string;
|
|
18
|
+
semanticType?: string;
|
|
19
|
+
definition?: Record<string, unknown>;
|
|
20
|
+
};
|