@oxygen-agent/cli 1.64.5 → 1.99.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/browser-login.js +1 -1
- package/dist/credentials.d.ts +13 -5
- package/dist/credentials.js +114 -19
- package/dist/http-client.d.ts +5 -1
- package/dist/http-client.js +23 -2
- package/dist/index.js +1417 -306
- package/dist/skills.d.ts +8 -2
- package/dist/skills.js +67 -17
- package/dist/update.js +2 -5
- package/dist/windows-shim.d.ts +7 -0
- package/dist/windows-shim.js +21 -0
- package/node_modules/@oxygen/shared/dist/file-import.d.ts +5 -0
- package/node_modules/@oxygen/shared/dist/file-import.js +156 -6
- package/node_modules/@oxygen/shared/dist/object-storage.d.ts +26 -0
- package/node_modules/@oxygen/shared/dist/object-storage.js +115 -0
- package/node_modules/@oxygen/shared/dist/version.d.ts +1 -1
- package/node_modules/@oxygen/shared/dist/version.js +1 -1
- package/node_modules/@oxygen/workflows/dist/index.d.ts +91 -0
- package/node_modules/@oxygen/workflows/dist/index.js +232 -1
- package/package.json +1 -1
package/dist/skills.d.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
|
-
import { execFileSync } from "node:child_process";
|
|
1
|
+
import { execFileSync, spawnSync } from "node:child_process";
|
|
2
2
|
export type ExecFileSyncLike = typeof execFileSync;
|
|
3
|
+
export type SpawnSyncLike = typeof spawnSync;
|
|
3
4
|
export type SkillsInstallOptions = {
|
|
4
5
|
json?: boolean;
|
|
5
6
|
apiUrl?: string;
|
|
@@ -29,13 +30,18 @@ export type AutomaticSkillsInstallResult = {
|
|
|
29
30
|
};
|
|
30
31
|
export declare function listAgentSkills(options: SkillsListOptions): Promise<Record<string, unknown>>;
|
|
31
32
|
export declare function doctorAgentSkills(options: SkillsDoctorOptions): Promise<Record<string, unknown>>;
|
|
32
|
-
export declare function installAgentSkills(
|
|
33
|
+
export declare function installAgentSkills(// skipcq: JS-R1005
|
|
34
|
+
options: SkillsInstallOptions, runtime?: {
|
|
33
35
|
env?: NodeJS.ProcessEnv;
|
|
36
|
+
platform?: NodeJS.Platform;
|
|
34
37
|
execFileSync?: ExecFileSyncLike;
|
|
38
|
+
spawnSync?: SpawnSyncLike;
|
|
35
39
|
}): Record<string, unknown>;
|
|
36
40
|
export declare function runAutomaticSkillsInstall(options?: {
|
|
37
41
|
apiUrl?: string;
|
|
38
42
|
env?: NodeJS.ProcessEnv;
|
|
43
|
+
platform?: NodeJS.Platform;
|
|
39
44
|
execFileSync?: ExecFileSyncLike;
|
|
45
|
+
spawnSync?: SpawnSyncLike;
|
|
40
46
|
}): AutomaticSkillsInstallResult;
|
|
41
47
|
export declare function skippedAutomaticSkillsInstall(reason: string, apiUrl?: string): AutomaticSkillsInstallResult;
|
package/dist/skills.js
CHANGED
|
@@ -1,9 +1,10 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { spawnSync } from "node:child_process";
|
|
2
2
|
import { existsSync, readdirSync } from "node:fs";
|
|
3
3
|
import { homedir } from "node:os";
|
|
4
4
|
import { join, resolve } from "node:path";
|
|
5
5
|
import { OxygenError, toFailure } from "@oxygen/shared";
|
|
6
6
|
import { defaultApiUrl, normalizeApiUrl } from "./credentials.js";
|
|
7
|
+
import { resolveCmdShimInvocation } from "./windows-shim.js";
|
|
7
8
|
const DEFAULT_SKILL_AGENTS = ["codex", "claude-code", "cursor"];
|
|
8
9
|
const SKILL_INDEX_PATH = "/.well-known/skills/index.json";
|
|
9
10
|
export async function listAgentSkills(options) {
|
|
@@ -52,32 +53,39 @@ export async function doctorAgentSkills(options) {
|
|
|
52
53
|
}
|
|
53
54
|
return data;
|
|
54
55
|
}
|
|
55
|
-
export function installAgentSkills(
|
|
56
|
+
export function installAgentSkills(// skipcq: JS-R1005
|
|
57
|
+
options, runtime = {}) {
|
|
56
58
|
const apiUrl = normalizeApiUrl(readOption(options.apiUrl) ?? defaultApiUrl());
|
|
57
59
|
const indexUrl = skillIndexUrl(apiUrl);
|
|
58
60
|
const agents = readWords(options.agents ?? DEFAULT_SKILL_AGENTS);
|
|
59
61
|
const skill = readOption(options.skill) ?? "*";
|
|
62
|
+
const env = runtime.env ?? process.env;
|
|
63
|
+
const platform = runtime.platform ?? process.platform;
|
|
60
64
|
const args = installerAddArgs(indexUrl, agents, skill);
|
|
61
65
|
if (!options.project)
|
|
62
66
|
args.push("--global");
|
|
63
67
|
if (options.copy)
|
|
64
68
|
args.push("--copy");
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
})
|
|
73
|
-
}
|
|
74
|
-
|
|
75
|
-
|
|
69
|
+
const invocation = resolveCmdShimInvocation("npx", args, platform, env);
|
|
70
|
+
const spawn = runtime.spawnSync ?? wrapExecFileSync(runtime.execFileSync) ?? spawnSync;
|
|
71
|
+
const result = spawn(invocation.executable, invocation.args, {
|
|
72
|
+
encoding: "utf8",
|
|
73
|
+
env,
|
|
74
|
+
stdio: ["ignore", "pipe", "pipe"],
|
|
75
|
+
windowsHide: true,
|
|
76
|
+
...(platform === "win32" ? { windowsVerbatimArguments: true } : {}),
|
|
77
|
+
});
|
|
78
|
+
const stdout = typeof result.stdout === "string" ? result.stdout : "";
|
|
79
|
+
const stderr = typeof result.stderr === "string" ? result.stderr : "";
|
|
80
|
+
if (result.error || (typeof result.status === "number" && result.status !== 0)) {
|
|
81
|
+
const message = result.error instanceof Error ? result.error.message : null;
|
|
76
82
|
throw new OxygenError("skills_install_failed", "Unable to install Oxygen agent skills.", {
|
|
77
83
|
details: {
|
|
78
84
|
index_url: indexUrl,
|
|
79
|
-
|
|
80
|
-
|
|
85
|
+
executable: invocation.label,
|
|
86
|
+
exit_code: typeof result.status === "number" ? result.status : null,
|
|
87
|
+
stdout: stdout.slice(0, 2000),
|
|
88
|
+
stderr: (stderr || message || "unknown").slice(0, 2000),
|
|
81
89
|
},
|
|
82
90
|
exitCode: 1,
|
|
83
91
|
});
|
|
@@ -88,12 +96,45 @@ export function installAgentSkills(options, runtime = {}) {
|
|
|
88
96
|
agents,
|
|
89
97
|
skill,
|
|
90
98
|
scope: options.project ? "project" : "global",
|
|
91
|
-
output,
|
|
99
|
+
output: stdout,
|
|
92
100
|
};
|
|
93
101
|
}
|
|
102
|
+
// Adapt the legacy `execFileSync` runtime hook into a spawnSync-shaped return so
|
|
103
|
+
// existing test fixtures (update.test.ts) keep working without a coordinated
|
|
104
|
+
// rewrite. New tests should inject `spawnSync` directly.
|
|
105
|
+
function wrapExecFileSync(exec) {
|
|
106
|
+
if (!exec)
|
|
107
|
+
return undefined;
|
|
108
|
+
return ((command, args, opts) => {
|
|
109
|
+
try {
|
|
110
|
+
const stdout = exec(command, args, opts);
|
|
111
|
+
return {
|
|
112
|
+
status: 0,
|
|
113
|
+
signal: null,
|
|
114
|
+
output: [],
|
|
115
|
+
pid: 0,
|
|
116
|
+
stdout: typeof stdout === "string" ? stdout : Buffer.isBuffer(stdout) ? stdout.toString("utf8") : "",
|
|
117
|
+
stderr: "",
|
|
118
|
+
};
|
|
119
|
+
}
|
|
120
|
+
catch (error) {
|
|
121
|
+
const failure = error;
|
|
122
|
+
return {
|
|
123
|
+
status: typeof failure.status === "number" ? failure.status : 1,
|
|
124
|
+
signal: null,
|
|
125
|
+
output: [],
|
|
126
|
+
pid: 0,
|
|
127
|
+
stdout: failure.stdout ?? "",
|
|
128
|
+
stderr: failure.stderr ?? failure.message ?? "",
|
|
129
|
+
error: error instanceof Error ? error : new Error(String(error)),
|
|
130
|
+
};
|
|
131
|
+
}
|
|
132
|
+
});
|
|
133
|
+
}
|
|
94
134
|
export function runAutomaticSkillsInstall(options = {}) {
|
|
95
135
|
const apiUrl = normalizeApiUrl(readOption(options.apiUrl) ?? defaultApiUrl());
|
|
96
136
|
const env = options.env ?? process.env;
|
|
137
|
+
const platform = options.platform ?? process.platform;
|
|
97
138
|
if (env.OXYGEN_SKIP_SKILLS === "1") {
|
|
98
139
|
return skippedAutomaticSkillsInstall("OXYGEN_SKIP_SKILLS=1", apiUrl);
|
|
99
140
|
}
|
|
@@ -104,9 +145,14 @@ export function runAutomaticSkillsInstall(options = {}) {
|
|
|
104
145
|
agents: DEFAULT_SKILL_AGENTS,
|
|
105
146
|
skill: "*",
|
|
106
147
|
json: true,
|
|
148
|
+
// `npx skills add` symlinks by default; on Windows that requires Developer
|
|
149
|
+
// Mode or admin elevation, so copy is the only reliable unattended path.
|
|
150
|
+
copy: platform === "win32",
|
|
107
151
|
}, {
|
|
108
152
|
env,
|
|
153
|
+
platform,
|
|
109
154
|
...(options.execFileSync ? { execFileSync: options.execFileSync } : {}),
|
|
155
|
+
...(options.spawnSync ? { spawnSync: options.spawnSync } : {}),
|
|
110
156
|
});
|
|
111
157
|
return {
|
|
112
158
|
attempted: true,
|
|
@@ -233,10 +279,13 @@ function parseSkillIndex(value) {
|
|
|
233
279
|
};
|
|
234
280
|
}
|
|
235
281
|
function inspectSkillsInstaller() {
|
|
236
|
-
const
|
|
282
|
+
const invocation = resolveCmdShimInvocation("npx", ["--version"], process.platform, process.env);
|
|
283
|
+
const result = spawnSync(invocation.executable, invocation.args, {
|
|
237
284
|
encoding: "utf8",
|
|
238
285
|
stdio: ["ignore", "pipe", "pipe"],
|
|
239
286
|
timeout: 5000,
|
|
287
|
+
windowsHide: true,
|
|
288
|
+
...(process.platform === "win32" ? { windowsVerbatimArguments: true } : {}),
|
|
240
289
|
});
|
|
241
290
|
const error = result.error instanceof Error ? result.error.message : null;
|
|
242
291
|
const stdout = typeof result.stdout === "string" ? result.stdout.trim() : "";
|
|
@@ -244,6 +293,7 @@ function inspectSkillsInstaller() {
|
|
|
244
293
|
return {
|
|
245
294
|
command_path: "npx",
|
|
246
295
|
check_args: ["--version"],
|
|
296
|
+
invocation: { executable: invocation.label, args: invocation.args },
|
|
247
297
|
available: result.status === 0 && !error,
|
|
248
298
|
version: stdout || null,
|
|
249
299
|
...(error ? { error } : {}),
|
package/dist/update.js
CHANGED
|
@@ -2,6 +2,7 @@ import { spawnSync } from "node:child_process";
|
|
|
2
2
|
import { fileURLToPath } from "node:url";
|
|
3
3
|
import { OXYGEN_VERSION, OxygenError } from "@oxygen/shared";
|
|
4
4
|
import { runAutomaticSkillsInstall, skippedAutomaticSkillsInstall, } from "./skills.js";
|
|
5
|
+
import { quoteWindowsCmdArg } from "./windows-shim.js";
|
|
5
6
|
const DEFAULT_CLI_PACKAGE_SPEC = "@oxygen-agent/cli@latest";
|
|
6
7
|
// skipcq: JS-R1005 — handles platform/install-mode/dry-run combinations end-to-end
|
|
7
8
|
export function updateCli(options, runtime = {}) {
|
|
@@ -53,6 +54,7 @@ export function updateCli(options, runtime = {}) {
|
|
|
53
54
|
updated: true,
|
|
54
55
|
skills_install: runAutomaticSkillsInstall({
|
|
55
56
|
env,
|
|
57
|
+
platform,
|
|
56
58
|
...(runtime.execFileSync ? { execFileSync: runtime.execFileSync } : {}),
|
|
57
59
|
}),
|
|
58
60
|
};
|
|
@@ -111,11 +113,6 @@ function resolveNpmInvocations(args, platform, env) {
|
|
|
111
113
|
label: shell,
|
|
112
114
|
}];
|
|
113
115
|
}
|
|
114
|
-
function quoteWindowsCmdArg(value) {
|
|
115
|
-
if (/^[A-Za-z0-9_@+=:,./\\-]+$/.test(value))
|
|
116
|
-
return value;
|
|
117
|
-
return `"${value.replace(/(["^&|<>()%])/g, "^$1")}"`;
|
|
118
|
-
}
|
|
119
116
|
function quoteForDisplay(value) {
|
|
120
117
|
if (/^[A-Za-z0-9_@+=:,./-]+$/.test(value))
|
|
121
118
|
return value;
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
export type CmdShimInvocation = {
|
|
2
|
+
executable: string;
|
|
3
|
+
args: string[];
|
|
4
|
+
label: string;
|
|
5
|
+
};
|
|
6
|
+
export declare function resolveCmdShimInvocation(command: string, args: string[], platform?: NodeJS.Platform, env?: NodeJS.ProcessEnv): CmdShimInvocation;
|
|
7
|
+
export declare function quoteWindowsCmdArg(value: string): string;
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
// Windows-aware command launcher. Node's `execFileSync`/`spawnSync` cannot
|
|
2
|
+
// resolve `.cmd`/`.bat` shims (like `npx.cmd`) from a bare command name, so on
|
|
3
|
+
// Windows we route through `cmd.exe /d /s /c <command> <args>` with
|
|
4
|
+
// `windowsVerbatimArguments: true`. Spawn options must enable that flag at the
|
|
5
|
+
// caller — see `update.ts` and `skills.ts` for the pattern.
|
|
6
|
+
export function resolveCmdShimInvocation(command, args, platform = process.platform, env = process.env) {
|
|
7
|
+
if (platform !== "win32") {
|
|
8
|
+
return { executable: command, args, label: command };
|
|
9
|
+
}
|
|
10
|
+
const shell = env.ComSpec?.trim() || "cmd.exe";
|
|
11
|
+
return {
|
|
12
|
+
executable: shell,
|
|
13
|
+
args: ["/d", "/s", "/c", [command, ...args].map(quoteWindowsCmdArg).join(" ")],
|
|
14
|
+
label: shell,
|
|
15
|
+
};
|
|
16
|
+
}
|
|
17
|
+
export function quoteWindowsCmdArg(value) {
|
|
18
|
+
if (/^[A-Za-z0-9_@+=:,./\\-]+$/.test(value))
|
|
19
|
+
return value;
|
|
20
|
+
return `"${value.replace(/(["^&|<>()%])/g, "^$1")}"`;
|
|
21
|
+
}
|
|
@@ -10,11 +10,16 @@ export type NewTableImportRows = {
|
|
|
10
10
|
rows: Record<string, unknown>[];
|
|
11
11
|
keyBySource: Record<string, string>;
|
|
12
12
|
};
|
|
13
|
+
export declare const MAX_BUFFERED_IMPORT_PARSE_BYTES: number;
|
|
13
14
|
export declare function inferRowsFileFormat(path: string): RowsFileFormat;
|
|
14
15
|
export declare function normalizeRowsFormat(value: string | undefined, fallback: RowsFileFormat): RowsFileFormat;
|
|
15
16
|
export declare function parseRowsFileBuffer(buffer: Buffer, format: RowsFileFormat, options?: {
|
|
16
17
|
sheet?: string;
|
|
17
18
|
}): Promise<Record<string, unknown>[]>;
|
|
19
|
+
export declare function iterateRowsFileBufferBatches(buffer: Buffer, format: RowsFileFormat, options?: {
|
|
20
|
+
sheet?: string;
|
|
21
|
+
batchSize?: number;
|
|
22
|
+
}): AsyncGenerator<Record<string, unknown>[]>;
|
|
18
23
|
export declare function parseRowsText(text: string, format: Exclude<RowsFileFormat, "xlsx">): Record<string, unknown>[];
|
|
19
24
|
export declare function inferImportColumnLabels(rows: Record<string, unknown>[]): string[];
|
|
20
25
|
export declare function normalizeRowsForNewTable(rows: Record<string, unknown>[]): NewTableImportRows;
|
|
@@ -3,6 +3,7 @@ import readXlsxFile from "read-excel-file/node";
|
|
|
3
3
|
import { inferImportColumnDataType, parseDateValueToIso, } from "./column-types.js";
|
|
4
4
|
import { OxygenError } from "./index.js";
|
|
5
5
|
const MAX_IDENTIFIER_LENGTH = 63;
|
|
6
|
+
export const MAX_BUFFERED_IMPORT_PARSE_BYTES = 10 * 1024 * 1024;
|
|
6
7
|
export function inferRowsFileFormat(path) {
|
|
7
8
|
const extension = extname(path).toLowerCase();
|
|
8
9
|
if (extension === ".jsonl" || extension === ".ndjson")
|
|
@@ -31,6 +32,20 @@ export async function parseRowsFileBuffer(buffer, format, options = {}) {
|
|
|
31
32
|
return await parseXlsxRows(buffer, options);
|
|
32
33
|
return parseRowsText(buffer.toString("utf8"), format);
|
|
33
34
|
}
|
|
35
|
+
export async function* iterateRowsFileBufferBatches(buffer, format, options = {}) {
|
|
36
|
+
const batchSize = normalizeBatchSize(options.batchSize);
|
|
37
|
+
if (format === "csv") {
|
|
38
|
+
yield* iterateCsvRowBatches(buffer.toString("utf8"), batchSize);
|
|
39
|
+
return;
|
|
40
|
+
}
|
|
41
|
+
if (format === "jsonl") {
|
|
42
|
+
yield* iterateJsonlRowBatches(buffer.toString("utf8"), batchSize);
|
|
43
|
+
return;
|
|
44
|
+
}
|
|
45
|
+
assertBufferedParseWithinLimit(buffer, format);
|
|
46
|
+
const rows = await parseRowsFileBuffer(buffer, format, options.sheet ? { sheet: options.sheet } : {});
|
|
47
|
+
yield* chunkRows(rows, batchSize);
|
|
48
|
+
}
|
|
34
49
|
export function parseRowsText(text, format) {
|
|
35
50
|
if (format === "json")
|
|
36
51
|
return normalizeRowObjects(parseJsonArray(text));
|
|
@@ -145,12 +160,13 @@ function parseJsonArray(text) {
|
|
|
145
160
|
return parsed;
|
|
146
161
|
}
|
|
147
162
|
function normalizeRowObjects(rows) {
|
|
148
|
-
return rows.map((row) =>
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
}
|
|
163
|
+
return rows.map((row) => normalizeRowObject(row));
|
|
164
|
+
}
|
|
165
|
+
function normalizeRowObject(row) {
|
|
166
|
+
if (!row || typeof row !== "object" || Array.isArray(row)) {
|
|
167
|
+
throw new OxygenError("invalid_rows", "Rows must be JSON objects.", { exitCode: 1 });
|
|
168
|
+
}
|
|
169
|
+
return row;
|
|
154
170
|
}
|
|
155
171
|
function parseCsvRows(text) {
|
|
156
172
|
const records = parseCsvRecords(text);
|
|
@@ -166,6 +182,140 @@ function normalizeCsvImportCell(value) {
|
|
|
166
182
|
return null;
|
|
167
183
|
return value;
|
|
168
184
|
}
|
|
185
|
+
function* iterateCsvRowBatches(text, batchSize) {
|
|
186
|
+
const state = {
|
|
187
|
+
header: null,
|
|
188
|
+
batch: [],
|
|
189
|
+
record: [],
|
|
190
|
+
field: "",
|
|
191
|
+
inQuotes: false,
|
|
192
|
+
};
|
|
193
|
+
for (let index = 0; index < text.length; index += 1) {
|
|
194
|
+
const result = applyCsvCharacter(state, text.charAt(index), text.charAt(index + 1));
|
|
195
|
+
if (result.skipNext)
|
|
196
|
+
index += 1;
|
|
197
|
+
if (!result.recordComplete)
|
|
198
|
+
continue;
|
|
199
|
+
const ready = appendCsvRecordToBatch(state, finishCsvRecord(state), batchSize);
|
|
200
|
+
if (ready)
|
|
201
|
+
yield ready;
|
|
202
|
+
}
|
|
203
|
+
if (state.field || state.record.length > 0) {
|
|
204
|
+
const ready = appendCsvRecordToBatch(state, finishCsvRecord(state), batchSize);
|
|
205
|
+
if (ready)
|
|
206
|
+
yield ready;
|
|
207
|
+
}
|
|
208
|
+
if (state.batch.length > 0)
|
|
209
|
+
yield state.batch;
|
|
210
|
+
}
|
|
211
|
+
function applyCsvCharacter(state, char, next) {
|
|
212
|
+
return state.inQuotes
|
|
213
|
+
? applyQuotedCsvCharacter(state, char, next)
|
|
214
|
+
: applyUnquotedCsvCharacter(state, char);
|
|
215
|
+
}
|
|
216
|
+
function applyQuotedCsvCharacter(state, char, next) {
|
|
217
|
+
if (char === "\"" && next === "\"") {
|
|
218
|
+
state.field += "\"";
|
|
219
|
+
return { recordComplete: false, skipNext: true };
|
|
220
|
+
}
|
|
221
|
+
if (char === "\"") {
|
|
222
|
+
state.inQuotes = false;
|
|
223
|
+
}
|
|
224
|
+
else {
|
|
225
|
+
state.field += char;
|
|
226
|
+
}
|
|
227
|
+
return { recordComplete: false, skipNext: false };
|
|
228
|
+
}
|
|
229
|
+
function applyUnquotedCsvCharacter(state, char) {
|
|
230
|
+
if (char === "\"") {
|
|
231
|
+
state.inQuotes = true;
|
|
232
|
+
return { recordComplete: false, skipNext: false };
|
|
233
|
+
}
|
|
234
|
+
if (char === ",") {
|
|
235
|
+
pushCsvField(state);
|
|
236
|
+
return { recordComplete: false, skipNext: false };
|
|
237
|
+
}
|
|
238
|
+
if (char === "\n") {
|
|
239
|
+
return { recordComplete: true, skipNext: false };
|
|
240
|
+
}
|
|
241
|
+
if (char !== "\r")
|
|
242
|
+
state.field += char;
|
|
243
|
+
return { recordComplete: false, skipNext: false };
|
|
244
|
+
}
|
|
245
|
+
function pushCsvField(state) {
|
|
246
|
+
state.record.push(state.field);
|
|
247
|
+
state.field = "";
|
|
248
|
+
}
|
|
249
|
+
function finishCsvRecord(state) {
|
|
250
|
+
pushCsvField(state);
|
|
251
|
+
const record = state.record;
|
|
252
|
+
state.record = [];
|
|
253
|
+
state.field = "";
|
|
254
|
+
return record;
|
|
255
|
+
}
|
|
256
|
+
function appendCsvRecordToBatch(state, record, batchSize) {
|
|
257
|
+
const row = csvRecordToRow(record, state.header);
|
|
258
|
+
if (state.header === null)
|
|
259
|
+
state.header = record;
|
|
260
|
+
if (row)
|
|
261
|
+
state.batch.push(row);
|
|
262
|
+
if (state.batch.length < batchSize)
|
|
263
|
+
return null;
|
|
264
|
+
const ready = state.batch;
|
|
265
|
+
state.batch = [];
|
|
266
|
+
return ready;
|
|
267
|
+
}
|
|
268
|
+
function csvRecordToRow(record, header) {
|
|
269
|
+
if (header === null)
|
|
270
|
+
return null;
|
|
271
|
+
if (!record.some((cell) => cell.trim()))
|
|
272
|
+
return null;
|
|
273
|
+
return Object.fromEntries(header.map((key, index) => [key, normalizeCsvImportCell(record[index])]));
|
|
274
|
+
}
|
|
275
|
+
function* iterateJsonlRowBatches(text, batchSize) {
|
|
276
|
+
let batch = [];
|
|
277
|
+
let lineStart = 0;
|
|
278
|
+
for (let index = 0; index < text.length; index += 1) {
|
|
279
|
+
if (text.charAt(index) !== "\n")
|
|
280
|
+
continue;
|
|
281
|
+
const line = text.slice(lineStart, index).replace(/\r$/, "").trim();
|
|
282
|
+
lineStart = index + 1;
|
|
283
|
+
if (!line)
|
|
284
|
+
continue;
|
|
285
|
+
batch.push(normalizeRowObject(JSON.parse(line)));
|
|
286
|
+
if (batch.length >= batchSize) {
|
|
287
|
+
yield batch;
|
|
288
|
+
batch = [];
|
|
289
|
+
}
|
|
290
|
+
}
|
|
291
|
+
const trailingLine = text.slice(lineStart).replace(/\r$/, "").trim();
|
|
292
|
+
if (trailingLine)
|
|
293
|
+
batch.push(normalizeRowObject(JSON.parse(trailingLine)));
|
|
294
|
+
if (batch.length > 0)
|
|
295
|
+
yield batch;
|
|
296
|
+
}
|
|
297
|
+
function* chunkRows(rows, batchSize) {
|
|
298
|
+
for (let index = 0; index < rows.length; index += batchSize) {
|
|
299
|
+
yield rows.slice(index, index + batchSize);
|
|
300
|
+
}
|
|
301
|
+
}
|
|
302
|
+
function normalizeBatchSize(value) {
|
|
303
|
+
if (!Number.isFinite(value) || value === undefined)
|
|
304
|
+
return 1000;
|
|
305
|
+
return Math.max(1, Math.trunc(value));
|
|
306
|
+
}
|
|
307
|
+
function assertBufferedParseWithinLimit(buffer, format) {
|
|
308
|
+
if (buffer.byteLength <= MAX_BUFFERED_IMPORT_PARSE_BYTES)
|
|
309
|
+
return;
|
|
310
|
+
throw new OxygenError("buffered_import_too_large", "Large JSON array and XLSX imports require buffered parsing; use CSV or JSONL for large staged imports.", {
|
|
311
|
+
details: {
|
|
312
|
+
format,
|
|
313
|
+
file_bytes: buffer.byteLength,
|
|
314
|
+
max_buffered_parse_bytes: MAX_BUFFERED_IMPORT_PARSE_BYTES,
|
|
315
|
+
},
|
|
316
|
+
exitCode: 1,
|
|
317
|
+
});
|
|
318
|
+
}
|
|
169
319
|
function parseCsvRecords(text) {
|
|
170
320
|
const records = [];
|
|
171
321
|
let record = [];
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
export declare function isObjectStorageConfigured(): boolean;
|
|
2
|
+
export declare function buildImportObjectKey(input: {
|
|
3
|
+
organizationId: string;
|
|
4
|
+
fileName?: string | null;
|
|
5
|
+
}): string;
|
|
6
|
+
export declare function isImportObjectKeyForOrganization(key: string, organizationId: string): boolean;
|
|
7
|
+
export type PresignedImportUpload = {
|
|
8
|
+
uploadUrl: string;
|
|
9
|
+
bucket: string;
|
|
10
|
+
storageKey: string;
|
|
11
|
+
contentLength: number;
|
|
12
|
+
provider: "s3";
|
|
13
|
+
expiresInSeconds: number;
|
|
14
|
+
};
|
|
15
|
+
export declare function presignImportUpload(input: {
|
|
16
|
+
organizationId: string;
|
|
17
|
+
fileName?: string | null;
|
|
18
|
+
contentType?: string | null;
|
|
19
|
+
contentLength: number;
|
|
20
|
+
}): Promise<PresignedImportUpload>;
|
|
21
|
+
export declare function downloadImportObject(input: {
|
|
22
|
+
storageKey: string;
|
|
23
|
+
}): Promise<Buffer>;
|
|
24
|
+
export declare function deleteImportObject(input: {
|
|
25
|
+
storageKey: string;
|
|
26
|
+
}): Promise<void>;
|
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
import { randomUUID } from "node:crypto";
|
|
2
|
+
import { DeleteObjectCommand, GetObjectCommand, PutObjectCommand, S3Client, } from "@aws-sdk/client-s3";
|
|
3
|
+
import { getSignedUrl } from "@aws-sdk/s3-request-presigner";
|
|
4
|
+
import { OxygenError } from "./index.js";
|
|
5
|
+
// S3-compatible object storage for large CSV/file imports. The CLI uploads the
|
|
6
|
+
// raw file straight to the bucket via a presigned PUT URL (bypassing Vercel's
|
|
7
|
+
// ~4.5MB request-body limit), then the Fly worker downloads it and COPY-loads
|
|
8
|
+
// it. Configured against Hetzner Object Storage (S3-compatible) via the
|
|
9
|
+
// OXYGEN_IMPORT_S3_* env vars; works with any S3-compatible endpoint.
|
|
10
|
+
const PRESIGN_EXPIRY_SECONDS = 900;
|
|
11
|
+
function readImportStorageConfig() {
|
|
12
|
+
const endpoint = process.env.OXYGEN_IMPORT_S3_ENDPOINT?.trim();
|
|
13
|
+
const bucket = process.env.OXYGEN_IMPORT_S3_BUCKET?.trim();
|
|
14
|
+
const accessKeyId = process.env.OXYGEN_IMPORT_S3_ACCESS_KEY_ID?.trim();
|
|
15
|
+
const secretAccessKey = process.env.OXYGEN_IMPORT_S3_SECRET_ACCESS_KEY?.trim();
|
|
16
|
+
if (!endpoint || !bucket || !accessKeyId || !secretAccessKey)
|
|
17
|
+
return null;
|
|
18
|
+
// Hetzner uses a location code (fsn1/nbg1/hel1) as its region; the S3 SDK
|
|
19
|
+
// only needs a non-empty string, so default to "auto" when unset.
|
|
20
|
+
const region = process.env.OXYGEN_IMPORT_S3_REGION?.trim() || "auto";
|
|
21
|
+
// Path-style addressing avoids bucket-in-hostname DNS/TLS edge cases on
|
|
22
|
+
// S3-compatible providers; default on, opt out with "false".
|
|
23
|
+
const forcePathStyle = process.env.OXYGEN_IMPORT_S3_FORCE_PATH_STYLE?.trim() !== "false";
|
|
24
|
+
return { endpoint, region, bucket, accessKeyId, secretAccessKey, forcePathStyle };
|
|
25
|
+
}
|
|
26
|
+
export function isObjectStorageConfigured() {
|
|
27
|
+
return readImportStorageConfig() !== null;
|
|
28
|
+
}
|
|
29
|
+
let cachedClient = null;
|
|
30
|
+
function resolveClient() {
|
|
31
|
+
const config = readImportStorageConfig();
|
|
32
|
+
if (!config) {
|
|
33
|
+
throw new OxygenError("object_storage_not_configured", "Import object storage (OXYGEN_IMPORT_S3_*) is not configured.", { exitCode: 1 });
|
|
34
|
+
}
|
|
35
|
+
const cacheKey = `${config.endpoint}|${config.region}|${config.accessKeyId}|${config.forcePathStyle}`;
|
|
36
|
+
if (!cachedClient || cachedClient.key !== cacheKey) {
|
|
37
|
+
cachedClient = {
|
|
38
|
+
key: cacheKey,
|
|
39
|
+
client: new S3Client({
|
|
40
|
+
endpoint: config.endpoint,
|
|
41
|
+
region: config.region,
|
|
42
|
+
forcePathStyle: config.forcePathStyle,
|
|
43
|
+
credentials: {
|
|
44
|
+
accessKeyId: config.accessKeyId,
|
|
45
|
+
secretAccessKey: config.secretAccessKey,
|
|
46
|
+
},
|
|
47
|
+
}),
|
|
48
|
+
};
|
|
49
|
+
}
|
|
50
|
+
return { client: cachedClient.client, config };
|
|
51
|
+
}
|
|
52
|
+
// Keys are namespaced by org so a tenant can only ever be handed (and the
|
|
53
|
+
// enqueue route only accepts) keys under its own prefix.
|
|
54
|
+
export function buildImportObjectKey(input) {
|
|
55
|
+
const safeName = sanitizeFileName(input.fileName) || "import";
|
|
56
|
+
return `imports/${input.organizationId}/${randomUUID()}/${safeName}`;
|
|
57
|
+
}
|
|
58
|
+
export function isImportObjectKeyForOrganization(key, organizationId) {
|
|
59
|
+
return key.startsWith(`imports/${organizationId}/`);
|
|
60
|
+
}
|
|
61
|
+
export async function presignImportUpload(input) {
|
|
62
|
+
const { client, config } = resolveClient();
|
|
63
|
+
const storageKey = buildImportObjectKey({
|
|
64
|
+
organizationId: input.organizationId,
|
|
65
|
+
fileName: input.fileName ?? null,
|
|
66
|
+
});
|
|
67
|
+
const command = new PutObjectCommand({
|
|
68
|
+
Bucket: config.bucket,
|
|
69
|
+
Key: storageKey,
|
|
70
|
+
ContentLength: input.contentLength,
|
|
71
|
+
...(input.contentType ? { ContentType: input.contentType } : {}),
|
|
72
|
+
});
|
|
73
|
+
const uploadUrl = await getSignedUrl(client, command, { expiresIn: PRESIGN_EXPIRY_SECONDS });
|
|
74
|
+
return {
|
|
75
|
+
uploadUrl,
|
|
76
|
+
bucket: config.bucket,
|
|
77
|
+
storageKey,
|
|
78
|
+
contentLength: input.contentLength,
|
|
79
|
+
provider: "s3",
|
|
80
|
+
expiresInSeconds: PRESIGN_EXPIRY_SECONDS,
|
|
81
|
+
};
|
|
82
|
+
}
|
|
83
|
+
export async function downloadImportObject(input) {
|
|
84
|
+
const { client, config } = resolveClient();
|
|
85
|
+
const result = await client.send(new GetObjectCommand({ Bucket: config.bucket, Key: input.storageKey }));
|
|
86
|
+
const body = result.Body;
|
|
87
|
+
if (!body) {
|
|
88
|
+
throw new OxygenError("import_object_missing", "Import object had no body.", {
|
|
89
|
+
details: { storage_key: input.storageKey },
|
|
90
|
+
exitCode: 1,
|
|
91
|
+
});
|
|
92
|
+
}
|
|
93
|
+
const transform = body.transformToByteArray;
|
|
94
|
+
if (typeof transform === "function") {
|
|
95
|
+
return Buffer.from(await transform.call(body));
|
|
96
|
+
}
|
|
97
|
+
return streamToBuffer(body);
|
|
98
|
+
}
|
|
99
|
+
export async function deleteImportObject(input) {
|
|
100
|
+
const { client, config } = resolveClient();
|
|
101
|
+
await client.send(new DeleteObjectCommand({ Bucket: config.bucket, Key: input.storageKey }));
|
|
102
|
+
}
|
|
103
|
+
function sanitizeFileName(fileName) {
|
|
104
|
+
if (!fileName)
|
|
105
|
+
return "";
|
|
106
|
+
const base = fileName.split(/[\\/]/).pop() ?? "";
|
|
107
|
+
return base.replace(/[^a-zA-Z0-9._-]/g, "_").slice(0, 120);
|
|
108
|
+
}
|
|
109
|
+
async function streamToBuffer(stream) {
|
|
110
|
+
const chunks = [];
|
|
111
|
+
for await (const chunk of stream) {
|
|
112
|
+
chunks.push(typeof chunk === "string" ? Buffer.from(chunk) : Buffer.from(chunk));
|
|
113
|
+
}
|
|
114
|
+
return Buffer.concat(chunks);
|
|
115
|
+
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
export declare const OXYGEN_VERSION = "1.
|
|
1
|
+
export declare const OXYGEN_VERSION = "1.99.1";
|
|
@@ -1 +1 @@
|
|
|
1
|
-
export const OXYGEN_VERSION = "1.
|
|
1
|
+
export const OXYGEN_VERSION = "1.99.1";
|