orizu 0.1.0 → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +173 -43
- package/dist/jsonl-stream.js +91 -0
- package/package.json +1 -1
- package/src/index.ts +192 -31
- package/src/jsonl-stream.ts +131 -0
package/dist/index.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
import { createHash, randomBytes } from 'crypto';
|
|
3
|
-
import { basename } from 'path';
|
|
3
|
+
import { basename, extname } from 'path';
|
|
4
4
|
import { createServer } from 'http';
|
|
5
5
|
import { readFileSync, statSync, writeFileSync } from 'fs';
|
|
6
6
|
import { spawn } from 'child_process';
|
|
@@ -8,8 +8,8 @@ import { createInterface } from 'readline/promises';
|
|
|
8
8
|
import { stdin as input, stdout as output } from 'process';
|
|
9
9
|
import { clearServerCredentials, getServerCredentials, saveServerCredentials } from './credentials.js';
|
|
10
10
|
import { parseDatasetFile } from './file-parser.js';
|
|
11
|
+
import { streamJsonlRowChunks } from './jsonl-stream.js';
|
|
11
12
|
import { parseDatasetReference } from './dataset-download.js';
|
|
12
|
-
import { extractErrorMessage } from './error-response.js';
|
|
13
13
|
import { parseGlobalFlags } from './global-flags.js';
|
|
14
14
|
import { authedFetch, getBaseUrl, resolveLoginBaseUrl, setGlobalFlags } from './http.js';
|
|
15
15
|
import { formatTaskCreateError } from './task-create-error.js';
|
|
@@ -24,6 +24,9 @@ function getArg(name) {
|
|
|
24
24
|
}
|
|
25
25
|
return cliArgs[index + 1];
|
|
26
26
|
}
|
|
27
|
+
function normalizeSlugInput(slug) {
|
|
28
|
+
return slug.trim().toLowerCase();
|
|
29
|
+
}
|
|
27
30
|
function isInteractiveTerminal() {
|
|
28
31
|
return Boolean(process.stdin.isTTY && process.stdout.isTTY);
|
|
29
32
|
}
|
|
@@ -124,7 +127,8 @@ async function fetchTeams() {
|
|
|
124
127
|
return data.teams;
|
|
125
128
|
}
|
|
126
129
|
async function fetchProjects(teamSlug) {
|
|
127
|
-
const
|
|
130
|
+
const normalizedTeamSlug = teamSlug ? normalizeSlugInput(teamSlug) : undefined;
|
|
131
|
+
const query = normalizedTeamSlug ? `?teamSlug=${encodeURIComponent(normalizedTeamSlug)}` : '';
|
|
128
132
|
const response = await authedFetch(`/api/cli/projects${query}`);
|
|
129
133
|
if (!response.ok) {
|
|
130
134
|
throw new Error(`Failed to fetch projects: ${await response.text()}`);
|
|
@@ -180,7 +184,7 @@ async function resolveProjectSlug(projectArg) {
|
|
|
180
184
|
if (segments.length !== 2 || !segments[0] || !segments[1]) {
|
|
181
185
|
throw new Error('Project must be in format teamSlug/projectSlug');
|
|
182
186
|
}
|
|
183
|
-
const [teamSlug, projectSlug] = segments;
|
|
187
|
+
const [teamSlug, projectSlug] = segments.map(normalizeSlugInput);
|
|
184
188
|
const matchedTeam = teams.find(team => team.slug === teamSlug);
|
|
185
189
|
if (!matchedTeam) {
|
|
186
190
|
console.error(`Team '${teamSlug}' not found in your accessible teams.`);
|
|
@@ -189,14 +193,14 @@ async function resolveProjectSlug(projectArg) {
|
|
|
189
193
|
const selectedProject = await promptSelect(`Select a project in ${selectedTeam.slug}`, projects, project => `${project.name} (${project.teamSlug}/${project.slug})`);
|
|
190
194
|
return `${selectedProject.teamSlug}/${selectedProject.slug}`;
|
|
191
195
|
}
|
|
192
|
-
const projects = await fetchProjects(
|
|
196
|
+
const projects = await fetchProjects(matchedTeam.slug);
|
|
193
197
|
const matchedProject = projects.find(project => project.slug === projectSlug);
|
|
194
198
|
if (!matchedProject) {
|
|
195
|
-
console.error(`Project '${projectSlug}' not found in team '${
|
|
196
|
-
const selectedProject = await promptSelect(`Select a project in ${
|
|
199
|
+
console.error(`Project '${projectSlug}' not found in team '${matchedTeam.slug}'.`);
|
|
200
|
+
const selectedProject = await promptSelect(`Select a project in ${matchedTeam.slug}`, projects, project => `${project.name} (${project.teamSlug}/${project.slug})`);
|
|
197
201
|
return `${selectedProject.teamSlug}/${selectedProject.slug}`;
|
|
198
202
|
}
|
|
199
|
-
return `${
|
|
203
|
+
return `${matchedTeam.slug}/${matchedProject.slug}`;
|
|
200
204
|
}
|
|
201
205
|
async function selectTaskIdInteractively() {
|
|
202
206
|
const team = await promptSelect('Select a team', await fetchTeams(), item => `${item.name} (${item.slug})`, { forcePrompt: true });
|
|
@@ -460,7 +464,7 @@ async function listTeams() {
|
|
|
460
464
|
}
|
|
461
465
|
async function resolveTeamSlug(teamSlugArg) {
|
|
462
466
|
if (teamSlugArg) {
|
|
463
|
-
return teamSlugArg;
|
|
467
|
+
return normalizeSlugInput(teamSlugArg);
|
|
464
468
|
}
|
|
465
469
|
const team = await promptSelect('Select a team', await fetchTeams(), item => `${item.name} (${item.slug})`, { forcePrompt: true });
|
|
466
470
|
return team.slug;
|
|
@@ -491,12 +495,14 @@ async function createTeam() {
|
|
|
491
495
|
console.log(`Created team: ${data.team.name} (${data.team.slug})`);
|
|
492
496
|
}
|
|
493
497
|
async function listProjects() {
|
|
494
|
-
const
|
|
498
|
+
const teamSlugArg = getArg('--team');
|
|
499
|
+
const teamSlug = teamSlugArg ? normalizeSlugInput(teamSlugArg) : null;
|
|
495
500
|
printProjects(await fetchProjects(teamSlug || undefined));
|
|
496
501
|
}
|
|
497
502
|
async function createProject() {
|
|
498
503
|
const name = getArg('--name');
|
|
499
|
-
|
|
504
|
+
const teamSlugArg = getArg('--team');
|
|
505
|
+
let teamSlug = teamSlugArg ? normalizeSlugInput(teamSlugArg) : null;
|
|
500
506
|
if (!name) {
|
|
501
507
|
throw new Error('Usage: orizu projects create --name <name> [--team <teamSlug>]');
|
|
502
508
|
}
|
|
@@ -700,7 +706,6 @@ async function createTask() {
|
|
|
700
706
|
...payload,
|
|
701
707
|
httpStatus: cliError.httpStatus,
|
|
702
708
|
}, null, 2));
|
|
703
|
-
process.exit(1);
|
|
704
709
|
}
|
|
705
710
|
throw cliError;
|
|
706
711
|
}
|
|
@@ -756,8 +761,8 @@ async function taskStatus() {
|
|
|
756
761
|
}
|
|
757
762
|
const response = await authedFetch(`/api/cli/tasks/${encodeURIComponent(taskId)}/status`);
|
|
758
763
|
if (!response.ok) {
|
|
764
|
+
const rawBody = await response.text();
|
|
759
765
|
if (hasArg('--json')) {
|
|
760
|
-
const rawBody = await response.text();
|
|
761
766
|
let errorPayload = { error: rawBody };
|
|
762
767
|
try {
|
|
763
768
|
const parsed = JSON.parse(rawBody);
|
|
@@ -772,9 +777,16 @@ async function taskStatus() {
|
|
|
772
777
|
...errorPayload,
|
|
773
778
|
httpStatus: response.status,
|
|
774
779
|
}, null, 2));
|
|
775
|
-
process.exit(1);
|
|
776
780
|
}
|
|
777
|
-
|
|
781
|
+
let errorMsg = rawBody;
|
|
782
|
+
try {
|
|
783
|
+
const parsed = JSON.parse(rawBody);
|
|
784
|
+
if (parsed && typeof parsed === 'object' && typeof parsed.error === 'string') {
|
|
785
|
+
errorMsg = parsed.error;
|
|
786
|
+
}
|
|
787
|
+
}
|
|
788
|
+
catch { /* use rawBody as-is */ }
|
|
789
|
+
throw new Error(`Failed to fetch task status: ${errorMsg}`);
|
|
778
790
|
}
|
|
779
791
|
const data = await parseJsonResponse(response, 'Task status');
|
|
780
792
|
if (hasArg('--json')) {
|
|
@@ -809,12 +821,7 @@ async function appDetail() {
|
|
|
809
821
|
throw new Error('Usage: orizu apps detail --app <appId> [--project <team/project>] [--json]');
|
|
810
822
|
}
|
|
811
823
|
const projectSlug = project || await resolveProjectSlug(null);
|
|
812
|
-
|
|
813
|
-
const app = apps.find(a => a.id === appId);
|
|
814
|
-
if (!app) {
|
|
815
|
-
throw new Error(`App '${appId}' not found in project '${projectSlug}'`);
|
|
816
|
-
}
|
|
817
|
-
// Re-fetch full detail from the list (it already returns full data)
|
|
824
|
+
// Single fetch — the apps endpoint already returns full detail (ALI-544)
|
|
818
825
|
const detailResponse = await authedFetch(`/api/cli/apps?project=${encodeURIComponent(projectSlug)}`);
|
|
819
826
|
if (!detailResponse.ok) {
|
|
820
827
|
throw new Error(`Failed to fetch app detail: ${await detailResponse.text()}`);
|
|
@@ -822,7 +829,7 @@ async function appDetail() {
|
|
|
822
829
|
const detailData = await parseJsonResponse(detailResponse, 'App detail');
|
|
823
830
|
const detail = detailData.apps.find(a => a.id === appId);
|
|
824
831
|
if (!detail) {
|
|
825
|
-
throw new Error(`App '${appId}' not found`);
|
|
832
|
+
throw new Error(`App '${appId}' not found in project '${projectSlug}'`);
|
|
826
833
|
}
|
|
827
834
|
if (hasArg('--json')) {
|
|
828
835
|
console.log(JSON.stringify(detail, null, 2));
|
|
@@ -884,7 +891,8 @@ async function removeTeamMember() {
|
|
|
884
891
|
console.log(`Removed team member ${member.email}`);
|
|
885
892
|
}
|
|
886
893
|
async function changeTeamMemberRole() {
|
|
887
|
-
const
|
|
894
|
+
const teamSlugArg = getArg('--team');
|
|
895
|
+
const teamSlug = teamSlugArg ? normalizeSlugInput(teamSlugArg) : null;
|
|
888
896
|
const email = getArg('--email');
|
|
889
897
|
const role = getArg('--role');
|
|
890
898
|
if (!teamSlug || !email || !role) {
|
|
@@ -908,17 +916,7 @@ async function changeTeamMemberRole() {
|
|
|
908
916
|
}
|
|
909
917
|
console.log(`Updated ${member.email} role to ${role}`);
|
|
910
918
|
}
|
|
911
|
-
async function
|
|
912
|
-
const projectArg = getArg('--project');
|
|
913
|
-
const fileArg = getArg('--file');
|
|
914
|
-
const name = getArg('--name');
|
|
915
|
-
if (!fileArg) {
|
|
916
|
-
throw new Error('Usage: orizu datasets upload --file <path> [--project <team/project>] [--name <name>]');
|
|
917
|
-
}
|
|
918
|
-
const file = expandHomePath(fileArg);
|
|
919
|
-
const project = await resolveProjectSlug(projectArg);
|
|
920
|
-
const { rows, sourceType } = parseDatasetFile(file);
|
|
921
|
-
const datasetName = name || basename(file);
|
|
919
|
+
async function createDatasetFromRows(project, name, sourceType, rows) {
|
|
922
920
|
const response = await authedFetch('/api/cli/datasets/upload', {
|
|
923
921
|
method: 'POST',
|
|
924
922
|
headers: {
|
|
@@ -926,7 +924,7 @@ async function uploadDataset() {
|
|
|
926
924
|
},
|
|
927
925
|
body: JSON.stringify({
|
|
928
926
|
projectSlug: project,
|
|
929
|
-
name
|
|
927
|
+
name,
|
|
930
928
|
rows,
|
|
931
929
|
sourceType,
|
|
932
930
|
}),
|
|
@@ -935,7 +933,83 @@ async function uploadDataset() {
|
|
|
935
933
|
const body = await parseJsonResponse(response, 'Dataset upload');
|
|
936
934
|
throw new Error(`Upload failed: ${body.error}`);
|
|
937
935
|
}
|
|
938
|
-
|
|
936
|
+
return parseJsonResponse(response, 'Dataset upload');
|
|
937
|
+
}
|
|
938
|
+
async function uploadJsonlDatasetInChunks(file, project, datasetName) {
|
|
939
|
+
let dataset = null;
|
|
940
|
+
let totalUploaded = 0;
|
|
941
|
+
let chunkIndex = 0;
|
|
942
|
+
const chunks = streamJsonlRowChunks(file)[Symbol.asyncIterator]();
|
|
943
|
+
while (true) {
|
|
944
|
+
let nextChunk;
|
|
945
|
+
try {
|
|
946
|
+
nextChunk = await chunks.next();
|
|
947
|
+
}
|
|
948
|
+
catch (error) {
|
|
949
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
950
|
+
if (!dataset) {
|
|
951
|
+
throw new Error(message);
|
|
952
|
+
}
|
|
953
|
+
throw new Error(`Upload stopped while reading the next JSONL chunk: ${message}\n` +
|
|
954
|
+
`Dataset ${dataset.name} (${dataset.id}) was created and ${totalUploaded} rows were uploaded. ` +
|
|
955
|
+
`Fix the file, remove the first ${totalUploaded} rows, and run ` +
|
|
956
|
+
`orizu datasets append --dataset ${dataset.id} --file <remaining-file>.`);
|
|
957
|
+
}
|
|
958
|
+
if (nextChunk.done) {
|
|
959
|
+
break;
|
|
960
|
+
}
|
|
961
|
+
const chunk = nextChunk.value;
|
|
962
|
+
chunkIndex += 1;
|
|
963
|
+
console.log(`Uploading chunk ${chunkIndex} (${chunk.length} rows)...`);
|
|
964
|
+
try {
|
|
965
|
+
if (!dataset) {
|
|
966
|
+
const data = await createDatasetFromRows(project, datasetName, 'jsonl', chunk);
|
|
967
|
+
dataset = data.dataset;
|
|
968
|
+
totalUploaded = data.dataset.rowCount;
|
|
969
|
+
continue;
|
|
970
|
+
}
|
|
971
|
+
const data = await appendChunk(dataset.id, chunk);
|
|
972
|
+
totalUploaded += data.appendedCount;
|
|
973
|
+
dataset = {
|
|
974
|
+
...dataset,
|
|
975
|
+
rowCount: data.dataset.rowCount,
|
|
976
|
+
};
|
|
977
|
+
}
|
|
978
|
+
catch (error) {
|
|
979
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
980
|
+
if (!dataset) {
|
|
981
|
+
throw new Error(message);
|
|
982
|
+
}
|
|
983
|
+
throw new Error(`Chunk ${chunkIndex} failed: ${message}\n` +
|
|
984
|
+
`Dataset ${dataset.name} (${dataset.id}) was created and ${totalUploaded} rows were uploaded. ` +
|
|
985
|
+
`To retry, remove the first ${totalUploaded} rows from your file and run ` +
|
|
986
|
+
`orizu datasets append --dataset ${dataset.id} --file <remaining-file>.`);
|
|
987
|
+
}
|
|
988
|
+
}
|
|
989
|
+
if (!dataset) {
|
|
990
|
+
throw new Error('Dataset file contains no rows');
|
|
991
|
+
}
|
|
992
|
+
console.log(`Uploaded dataset ${dataset.name} (${dataset.id}) with ${dataset.rowCount} rows.`);
|
|
993
|
+
if (dataset.url) {
|
|
994
|
+
console.log(`View dataset: ${formatTerminalLink(dataset.url)}`);
|
|
995
|
+
}
|
|
996
|
+
}
|
|
997
|
+
async function uploadDataset() {
|
|
998
|
+
const projectArg = getArg('--project');
|
|
999
|
+
const fileArg = getArg('--file');
|
|
1000
|
+
const name = getArg('--name');
|
|
1001
|
+
if (!fileArg) {
|
|
1002
|
+
throw new Error('Usage: orizu datasets upload --file <path> [--project <team/project>] [--name <name>]');
|
|
1003
|
+
}
|
|
1004
|
+
const file = expandHomePath(fileArg);
|
|
1005
|
+
const project = await resolveProjectSlug(projectArg);
|
|
1006
|
+
const datasetName = name || basename(file);
|
|
1007
|
+
if (extname(file).toLowerCase() === '.jsonl') {
|
|
1008
|
+
await uploadJsonlDatasetInChunks(file, project, datasetName);
|
|
1009
|
+
return;
|
|
1010
|
+
}
|
|
1011
|
+
const { rows, sourceType } = parseDatasetFile(file);
|
|
1012
|
+
const data = await createDatasetFromRows(project, datasetName, sourceType, rows);
|
|
939
1013
|
console.log(`Uploaded dataset ${data.dataset.name} (${data.dataset.id}) with ${data.dataset.rowCount} rows.`);
|
|
940
1014
|
if (data.dataset.url) {
|
|
941
1015
|
console.log(`View dataset: ${formatTerminalLink(data.dataset.url)}`);
|
|
@@ -992,6 +1066,45 @@ async function appendChunk(datasetId, rows) {
|
|
|
992
1066
|
}
|
|
993
1067
|
return parseJsonResponse(response, 'Dataset append');
|
|
994
1068
|
}
|
|
1069
|
+
async function appendJsonlDatasetRowsInChunks(datasetId, file) {
|
|
1070
|
+
let totalAppended = 0;
|
|
1071
|
+
let lastResult = null;
|
|
1072
|
+
let chunkIndex = 0;
|
|
1073
|
+
const chunks = streamJsonlRowChunks(file)[Symbol.asyncIterator]();
|
|
1074
|
+
while (true) {
|
|
1075
|
+
let nextChunk;
|
|
1076
|
+
try {
|
|
1077
|
+
nextChunk = await chunks.next();
|
|
1078
|
+
}
|
|
1079
|
+
catch (error) {
|
|
1080
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
1081
|
+
throw new Error(`Append stopped while reading the next JSONL chunk: ${message}\n` +
|
|
1082
|
+
`${totalAppended} rows from ${chunkIndex} chunk(s) were already appended. ` +
|
|
1083
|
+
`Fix the file, remove the first ${totalAppended} rows, and re-run the command.`);
|
|
1084
|
+
}
|
|
1085
|
+
if (nextChunk.done) {
|
|
1086
|
+
break;
|
|
1087
|
+
}
|
|
1088
|
+
const chunk = nextChunk.value;
|
|
1089
|
+
chunkIndex += 1;
|
|
1090
|
+
console.log(`Uploading chunk ${chunkIndex} (${chunk.length} rows)...`);
|
|
1091
|
+
try {
|
|
1092
|
+
const data = await appendChunk(datasetId, chunk);
|
|
1093
|
+
totalAppended += data.appendedCount;
|
|
1094
|
+
lastResult = data;
|
|
1095
|
+
}
|
|
1096
|
+
catch (error) {
|
|
1097
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
1098
|
+
throw new Error(`Chunk ${chunkIndex} failed: ${message}\n` +
|
|
1099
|
+
`${totalAppended} rows from ${chunkIndex - 1} chunk(s) were already appended. ` +
|
|
1100
|
+
`To retry, remove the first ${totalAppended} rows from your file and re-run the command.`);
|
|
1101
|
+
}
|
|
1102
|
+
}
|
|
1103
|
+
if (!lastResult) {
|
|
1104
|
+
throw new Error('Dataset append file must contain at least one row');
|
|
1105
|
+
}
|
|
1106
|
+
console.log(`Appended ${totalAppended} rows to dataset ${lastResult.dataset.name} (${lastResult.dataset.id}). New row count: ${lastResult.dataset.rowCount}`);
|
|
1107
|
+
}
|
|
995
1108
|
async function appendDatasetRows() {
|
|
996
1109
|
const projectArg = getArg('--project');
|
|
997
1110
|
const datasetInput = getDatasetReferenceInput();
|
|
@@ -1008,18 +1121,35 @@ async function appendDatasetRows() {
|
|
|
1008
1121
|
datasetId = selected.datasetId;
|
|
1009
1122
|
}
|
|
1010
1123
|
const file = expandHomePath(fileArg);
|
|
1011
|
-
|
|
1012
|
-
|
|
1013
|
-
|
|
1014
|
-
|
|
1015
|
-
|
|
1016
|
-
|
|
1124
|
+
if (extname(file).toLowerCase() === '.jsonl') {
|
|
1125
|
+
await appendJsonlDatasetRowsInChunks(datasetId, file);
|
|
1126
|
+
return;
|
|
1127
|
+
}
|
|
1128
|
+
// Check file size before reading to prevent OOM on large files (ALI-565).
|
|
1129
|
+
// Wrap statSync in try/catch so missing/inaccessible files get friendly
|
|
1130
|
+
// errors instead of raw Node.js ENOENT/EPERM (ALI-554).
|
|
1131
|
+
let fileSizeBytes;
|
|
1132
|
+
try {
|
|
1133
|
+
fileSizeBytes = statSync(file).size;
|
|
1134
|
+
}
|
|
1135
|
+
catch (error) {
|
|
1136
|
+
const maybeError = error;
|
|
1137
|
+
if (maybeError.code === 'ENOENT') {
|
|
1138
|
+
throw new Error(`File not found: ${file}. Check the path and filename, then retry.`);
|
|
1139
|
+
}
|
|
1140
|
+
if (maybeError.code === 'EPERM' || maybeError.code === 'EACCES') {
|
|
1141
|
+
throw new Error(`Cannot read file: ${file}. Grant folder permission to your terminal app and retry.`);
|
|
1142
|
+
}
|
|
1143
|
+
throw new Error(`Failed to access file ${file}: ${maybeError.message}`);
|
|
1017
1144
|
}
|
|
1018
|
-
const fileSizeBytes = statSync(file).size;
|
|
1019
1145
|
if (fileSizeBytes > MAX_INPUT_FILE_SIZE_BYTES) {
|
|
1020
1146
|
const sizeMb = (fileSizeBytes / (1024 * 1024)).toFixed(1);
|
|
1021
1147
|
throw new Error(`Input file is ${sizeMb} MB, which exceeds the 50 MB limit. Split the file into smaller parts and append each separately.`);
|
|
1022
1148
|
}
|
|
1149
|
+
const { rows } = parseDatasetFile(file);
|
|
1150
|
+
if (!Array.isArray(rows) || rows.length === 0) {
|
|
1151
|
+
throw new Error('Dataset append file must contain at least one row');
|
|
1152
|
+
}
|
|
1023
1153
|
if (rows.length <= APPEND_CHUNK_SIZE_ROWS) {
|
|
1024
1154
|
const data = await appendChunk(datasetId, rows);
|
|
1025
1155
|
console.log(`Appended ${data.appendedCount} rows to dataset ${data.dataset.name} (${data.dataset.id}). New row count: ${data.dataset.rowCount}`);
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
import { createReadStream } from 'fs';
|
|
2
|
+
import { createInterface } from 'readline';
|
|
3
|
+
export const JSONL_UPLOAD_CHUNK_MAX_BYTES = 2 * 1024 * 1024;
|
|
4
|
+
function isObjectRow(value) {
|
|
5
|
+
return typeof value === 'object' && value !== null && !Array.isArray(value);
|
|
6
|
+
}
|
|
7
|
+
function rowsPayloadByteLength(rowCount, rowsByteLength) {
|
|
8
|
+
return Buffer.byteLength('{"rows":[]}', 'utf8') +
|
|
9
|
+
rowsByteLength +
|
|
10
|
+
Math.max(0, rowCount - 1);
|
|
11
|
+
}
|
|
12
|
+
function formatMb(byteLength) {
|
|
13
|
+
return (byteLength / (1024 * 1024)).toFixed(1);
|
|
14
|
+
}
|
|
15
|
+
function mapReadError(filePath, error) {
|
|
16
|
+
const maybeError = error;
|
|
17
|
+
if (!maybeError.code && error instanceof Error) {
|
|
18
|
+
return error;
|
|
19
|
+
}
|
|
20
|
+
if (maybeError.code === 'ENOENT') {
|
|
21
|
+
return new Error(`File not found: ${filePath}. Check the path and filename, then retry.`);
|
|
22
|
+
}
|
|
23
|
+
if (maybeError.code === 'EPERM' || maybeError.code === 'EACCES') {
|
|
24
|
+
return new Error(`Cannot read file: ${filePath}. macOS may be blocking access to this folder (for example Downloads). Grant folder permission to your terminal app and retry.`);
|
|
25
|
+
}
|
|
26
|
+
return new Error(`Failed to read file ${filePath}: ${maybeError.message}`);
|
|
27
|
+
}
|
|
28
|
+
export function estimateRowsPayloadByteLength(rows) {
|
|
29
|
+
const rowsByteLength = rows.reduce((total, row) => {
|
|
30
|
+
return total + Buffer.byteLength(JSON.stringify(row), 'utf8');
|
|
31
|
+
}, 0);
|
|
32
|
+
return rowsPayloadByteLength(rows.length, rowsByteLength);
|
|
33
|
+
}
|
|
34
|
+
export async function* streamJsonlRowChunks(filePath, options = {}) {
|
|
35
|
+
const maxPayloadBytes = options.maxPayloadBytes ?? JSONL_UPLOAD_CHUNK_MAX_BYTES;
|
|
36
|
+
let chunk = [];
|
|
37
|
+
let chunkRowsByteLength = 0;
|
|
38
|
+
let parsedRowCount = 0;
|
|
39
|
+
let lineNumber = 0;
|
|
40
|
+
const stream = createReadStream(filePath, { encoding: 'utf8' });
|
|
41
|
+
const lines = createInterface({
|
|
42
|
+
input: stream,
|
|
43
|
+
crlfDelay: Infinity,
|
|
44
|
+
});
|
|
45
|
+
try {
|
|
46
|
+
for await (const line of lines) {
|
|
47
|
+
lineNumber += 1;
|
|
48
|
+
const trimmed = line.trim();
|
|
49
|
+
if (!trimmed) {
|
|
50
|
+
continue;
|
|
51
|
+
}
|
|
52
|
+
let parsed;
|
|
53
|
+
try {
|
|
54
|
+
parsed = JSON.parse(trimmed);
|
|
55
|
+
}
|
|
56
|
+
catch {
|
|
57
|
+
throw new Error(`Invalid JSONL at line ${lineNumber}`);
|
|
58
|
+
}
|
|
59
|
+
if (!isObjectRow(parsed)) {
|
|
60
|
+
throw new Error(`Row ${lineNumber} must be a JSON object`);
|
|
61
|
+
}
|
|
62
|
+
const jsonByteLength = Buffer.byteLength(JSON.stringify(parsed), 'utf8');
|
|
63
|
+
const singleRowPayloadBytes = rowsPayloadByteLength(1, jsonByteLength);
|
|
64
|
+
if (singleRowPayloadBytes > maxPayloadBytes) {
|
|
65
|
+
throw new Error(`Row ${lineNumber} is ${formatMb(singleRowPayloadBytes)} MB after JSON encoding, which exceeds the ${formatMb(maxPayloadBytes)} MB upload chunk limit`);
|
|
66
|
+
}
|
|
67
|
+
const nextPayloadBytes = rowsPayloadByteLength(chunk.length + 1, chunkRowsByteLength + jsonByteLength);
|
|
68
|
+
if (chunk.length > 0 && nextPayloadBytes > maxPayloadBytes) {
|
|
69
|
+
yield chunk.map(item => item.row);
|
|
70
|
+
chunk = [];
|
|
71
|
+
chunkRowsByteLength = 0;
|
|
72
|
+
}
|
|
73
|
+
chunk.push({ row: parsed, jsonByteLength });
|
|
74
|
+
chunkRowsByteLength += jsonByteLength;
|
|
75
|
+
parsedRowCount += 1;
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
catch (error) {
|
|
79
|
+
throw mapReadError(filePath, error);
|
|
80
|
+
}
|
|
81
|
+
finally {
|
|
82
|
+
lines.close();
|
|
83
|
+
stream.destroy();
|
|
84
|
+
}
|
|
85
|
+
if (chunk.length > 0) {
|
|
86
|
+
yield chunk.map(item => item.row);
|
|
87
|
+
}
|
|
88
|
+
if (parsedRowCount === 0) {
|
|
89
|
+
throw new Error('Dataset file contains no rows');
|
|
90
|
+
}
|
|
91
|
+
}
|
package/package.json
CHANGED
package/src/index.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
import { createHash, randomBytes } from 'crypto'
|
|
3
|
-
import { basename } from 'path'
|
|
3
|
+
import { basename, extname } from 'path'
|
|
4
4
|
import { createServer } from 'http'
|
|
5
5
|
import { readFileSync, statSync, writeFileSync } from 'fs'
|
|
6
6
|
import { spawn } from 'child_process'
|
|
@@ -8,8 +8,8 @@ import { createInterface } from 'readline/promises'
|
|
|
8
8
|
import { stdin as input, stdout as output } from 'process'
|
|
9
9
|
import { clearServerCredentials, getServerCredentials, saveServerCredentials } from './credentials.js'
|
|
10
10
|
import { parseDatasetFile } from './file-parser.js'
|
|
11
|
+
import { streamJsonlRowChunks } from './jsonl-stream.js'
|
|
11
12
|
import { parseDatasetReference } from './dataset-download.js'
|
|
12
|
-
import { extractErrorMessage } from './error-response.js'
|
|
13
13
|
import { parseGlobalFlags } from './global-flags.js'
|
|
14
14
|
import { authedFetch, getBaseUrl, resolveLoginBaseUrl, setGlobalFlags } from './http.js'
|
|
15
15
|
import { formatTaskCreateError } from './task-create-error.js'
|
|
@@ -67,6 +67,18 @@ interface DatasetSummary {
|
|
|
67
67
|
teamSlug: string
|
|
68
68
|
}
|
|
69
69
|
|
|
70
|
+
type DatasetUploadSourceType = 'csv' | 'json' | 'jsonl'
|
|
71
|
+
|
|
72
|
+
interface DatasetUploadResponse {
|
|
73
|
+
dataset: {
|
|
74
|
+
id: string
|
|
75
|
+
name: string
|
|
76
|
+
rowCount: number
|
|
77
|
+
sourceType: string
|
|
78
|
+
url?: string
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
|
|
70
82
|
interface TeamMember {
|
|
71
83
|
id: string
|
|
72
84
|
user_id: string | null
|
|
@@ -122,6 +134,10 @@ function getArg(name: string): string | null {
|
|
|
122
134
|
return cliArgs[index + 1]
|
|
123
135
|
}
|
|
124
136
|
|
|
137
|
+
function normalizeSlugInput(slug: string): string {
|
|
138
|
+
return slug.trim().toLowerCase()
|
|
139
|
+
}
|
|
140
|
+
|
|
125
141
|
function isInteractiveTerminal() {
|
|
126
142
|
return Boolean(process.stdin.isTTY && process.stdout.isTTY)
|
|
127
143
|
}
|
|
@@ -252,7 +268,8 @@ async function fetchTeams(): Promise<Team[]> {
|
|
|
252
268
|
}
|
|
253
269
|
|
|
254
270
|
async function fetchProjects(teamSlug?: string): Promise<Project[]> {
|
|
255
|
-
const
|
|
271
|
+
const normalizedTeamSlug = teamSlug ? normalizeSlugInput(teamSlug) : undefined
|
|
272
|
+
const query = normalizedTeamSlug ? `?teamSlug=${encodeURIComponent(normalizedTeamSlug)}` : ''
|
|
256
273
|
const response = await authedFetch(`/api/cli/projects${query}`)
|
|
257
274
|
if (!response.ok) {
|
|
258
275
|
throw new Error(`Failed to fetch projects: ${await response.text()}`)
|
|
@@ -333,7 +350,7 @@ async function resolveProjectSlug(projectArg: string | null): Promise<string> {
|
|
|
333
350
|
if (segments.length !== 2 || !segments[0] || !segments[1]) {
|
|
334
351
|
throw new Error('Project must be in format teamSlug/projectSlug')
|
|
335
352
|
}
|
|
336
|
-
const [teamSlug, projectSlug] = segments
|
|
353
|
+
const [teamSlug, projectSlug] = segments.map(normalizeSlugInput)
|
|
337
354
|
|
|
338
355
|
const matchedTeam = teams.find(team => team.slug === teamSlug)
|
|
339
356
|
if (!matchedTeam) {
|
|
@@ -354,13 +371,13 @@ async function resolveProjectSlug(projectArg: string | null): Promise<string> {
|
|
|
354
371
|
return `${selectedProject.teamSlug}/${selectedProject.slug}`
|
|
355
372
|
}
|
|
356
373
|
|
|
357
|
-
const projects = await fetchProjects(
|
|
374
|
+
const projects = await fetchProjects(matchedTeam.slug)
|
|
358
375
|
const matchedProject = projects.find(project => project.slug === projectSlug)
|
|
359
376
|
|
|
360
377
|
if (!matchedProject) {
|
|
361
|
-
console.error(`Project '${projectSlug}' not found in team '${
|
|
378
|
+
console.error(`Project '${projectSlug}' not found in team '${matchedTeam.slug}'.`)
|
|
362
379
|
const selectedProject = await promptSelect(
|
|
363
|
-
`Select a project in ${
|
|
380
|
+
`Select a project in ${matchedTeam.slug}`,
|
|
364
381
|
projects,
|
|
365
382
|
project => `${project.name} (${project.teamSlug}/${project.slug})`
|
|
366
383
|
)
|
|
@@ -368,7 +385,7 @@ async function resolveProjectSlug(projectArg: string | null): Promise<string> {
|
|
|
368
385
|
return `${selectedProject.teamSlug}/${selectedProject.slug}`
|
|
369
386
|
}
|
|
370
387
|
|
|
371
|
-
return `${
|
|
388
|
+
return `${matchedTeam.slug}/${matchedProject.slug}`
|
|
372
389
|
}
|
|
373
390
|
|
|
374
391
|
async function selectTaskIdInteractively(): Promise<string> {
|
|
@@ -745,7 +762,7 @@ async function listTeams() {
|
|
|
745
762
|
|
|
746
763
|
async function resolveTeamSlug(teamSlugArg: string | null): Promise<string> {
|
|
747
764
|
if (teamSlugArg) {
|
|
748
|
-
return teamSlugArg
|
|
765
|
+
return normalizeSlugInput(teamSlugArg)
|
|
749
766
|
}
|
|
750
767
|
|
|
751
768
|
const team = await promptSelect(
|
|
@@ -789,13 +806,15 @@ async function createTeam() {
|
|
|
789
806
|
}
|
|
790
807
|
|
|
791
808
|
async function listProjects() {
|
|
792
|
-
const
|
|
809
|
+
const teamSlugArg = getArg('--team')
|
|
810
|
+
const teamSlug = teamSlugArg ? normalizeSlugInput(teamSlugArg) : null
|
|
793
811
|
printProjects(await fetchProjects(teamSlug || undefined))
|
|
794
812
|
}
|
|
795
813
|
|
|
796
814
|
async function createProject() {
|
|
797
815
|
const name = getArg('--name')
|
|
798
|
-
|
|
816
|
+
const teamSlugArg = getArg('--team')
|
|
817
|
+
let teamSlug = teamSlugArg ? normalizeSlugInput(teamSlugArg) : null
|
|
799
818
|
|
|
800
819
|
if (!name) {
|
|
801
820
|
throw new Error('Usage: orizu projects create --name <name> [--team <teamSlug>]')
|
|
@@ -1308,7 +1327,8 @@ async function removeTeamMember() {
|
|
|
1308
1327
|
}
|
|
1309
1328
|
|
|
1310
1329
|
async function changeTeamMemberRole() {
|
|
1311
|
-
const
|
|
1330
|
+
const teamSlugArg = getArg('--team')
|
|
1331
|
+
const teamSlug = teamSlugArg ? normalizeSlugInput(teamSlugArg) : null
|
|
1312
1332
|
const email = getArg('--email')
|
|
1313
1333
|
const role = getArg('--role')
|
|
1314
1334
|
if (!teamSlug || !email || !role) {
|
|
@@ -1340,21 +1360,12 @@ async function changeTeamMemberRole() {
|
|
|
1340
1360
|
console.log(`Updated ${member.email} role to ${role}`)
|
|
1341
1361
|
}
|
|
1342
1362
|
|
|
1343
|
-
async function
|
|
1344
|
-
|
|
1345
|
-
|
|
1346
|
-
|
|
1347
|
-
|
|
1348
|
-
|
|
1349
|
-
throw new Error('Usage: orizu datasets upload --file <path> [--project <team/project>] [--name <name>]')
|
|
1350
|
-
}
|
|
1351
|
-
|
|
1352
|
-
const file = expandHomePath(fileArg)
|
|
1353
|
-
const project = await resolveProjectSlug(projectArg)
|
|
1354
|
-
|
|
1355
|
-
const { rows, sourceType } = parseDatasetFile(file)
|
|
1356
|
-
const datasetName = name || basename(file)
|
|
1357
|
-
|
|
1363
|
+
async function createDatasetFromRows(
|
|
1364
|
+
project: string,
|
|
1365
|
+
name: string,
|
|
1366
|
+
sourceType: DatasetUploadSourceType,
|
|
1367
|
+
rows: Array<Record<string, unknown>>
|
|
1368
|
+
): Promise<DatasetUploadResponse> {
|
|
1358
1369
|
const response = await authedFetch('/api/cli/datasets/upload', {
|
|
1359
1370
|
method: 'POST',
|
|
1360
1371
|
headers: {
|
|
@@ -1362,7 +1373,7 @@ async function uploadDataset() {
|
|
|
1362
1373
|
},
|
|
1363
1374
|
body: JSON.stringify({
|
|
1364
1375
|
projectSlug: project,
|
|
1365
|
-
name
|
|
1376
|
+
name,
|
|
1366
1377
|
rows,
|
|
1367
1378
|
sourceType,
|
|
1368
1379
|
}),
|
|
@@ -1373,9 +1384,104 @@ async function uploadDataset() {
|
|
|
1373
1384
|
throw new Error(`Upload failed: ${body.error}`)
|
|
1374
1385
|
}
|
|
1375
1386
|
|
|
1376
|
-
|
|
1377
|
-
|
|
1378
|
-
|
|
1387
|
+
return parseJsonResponse<DatasetUploadResponse>(response, 'Dataset upload')
|
|
1388
|
+
}
|
|
1389
|
+
|
|
1390
|
+
async function uploadJsonlDatasetInChunks(
|
|
1391
|
+
file: string,
|
|
1392
|
+
project: string,
|
|
1393
|
+
datasetName: string
|
|
1394
|
+
) {
|
|
1395
|
+
let dataset: DatasetUploadResponse['dataset'] | null = null
|
|
1396
|
+
let totalUploaded = 0
|
|
1397
|
+
let chunkIndex = 0
|
|
1398
|
+
const chunks = streamJsonlRowChunks(file)[Symbol.asyncIterator]()
|
|
1399
|
+
|
|
1400
|
+
while (true) {
|
|
1401
|
+
let nextChunk: IteratorResult<Array<Record<string, unknown>>>
|
|
1402
|
+
try {
|
|
1403
|
+
nextChunk = await chunks.next()
|
|
1404
|
+
} catch (error) {
|
|
1405
|
+
const message = error instanceof Error ? error.message : String(error)
|
|
1406
|
+
if (!dataset) {
|
|
1407
|
+
throw new Error(message)
|
|
1408
|
+
}
|
|
1409
|
+
|
|
1410
|
+
throw new Error(
|
|
1411
|
+
`Upload stopped while reading the next JSONL chunk: ${message}\n` +
|
|
1412
|
+
`Dataset ${dataset.name} (${dataset.id}) was created and ${totalUploaded} rows were uploaded. ` +
|
|
1413
|
+
`Fix the file, remove the first ${totalUploaded} rows, and run ` +
|
|
1414
|
+
`orizu datasets append --dataset ${dataset.id} --file <remaining-file>.`
|
|
1415
|
+
)
|
|
1416
|
+
}
|
|
1417
|
+
|
|
1418
|
+
if (nextChunk.done) {
|
|
1419
|
+
break
|
|
1420
|
+
}
|
|
1421
|
+
|
|
1422
|
+
const chunk = nextChunk.value
|
|
1423
|
+
chunkIndex += 1
|
|
1424
|
+
console.log(`Uploading chunk ${chunkIndex} (${chunk.length} rows)...`)
|
|
1425
|
+
|
|
1426
|
+
try {
|
|
1427
|
+
if (!dataset) {
|
|
1428
|
+
const data = await createDatasetFromRows(project, datasetName, 'jsonl', chunk)
|
|
1429
|
+
dataset = data.dataset
|
|
1430
|
+
totalUploaded = data.dataset.rowCount
|
|
1431
|
+
continue
|
|
1432
|
+
}
|
|
1433
|
+
|
|
1434
|
+
const data = await appendChunk(dataset.id, chunk)
|
|
1435
|
+
totalUploaded += data.appendedCount
|
|
1436
|
+
dataset = {
|
|
1437
|
+
...dataset,
|
|
1438
|
+
rowCount: data.dataset.rowCount,
|
|
1439
|
+
}
|
|
1440
|
+
} catch (error) {
|
|
1441
|
+
const message = error instanceof Error ? error.message : String(error)
|
|
1442
|
+
if (!dataset) {
|
|
1443
|
+
throw new Error(message)
|
|
1444
|
+
}
|
|
1445
|
+
|
|
1446
|
+
throw new Error(
|
|
1447
|
+
`Chunk ${chunkIndex} failed: ${message}\n` +
|
|
1448
|
+
`Dataset ${dataset.name} (${dataset.id}) was created and ${totalUploaded} rows were uploaded. ` +
|
|
1449
|
+
`To retry, remove the first ${totalUploaded} rows from your file and run ` +
|
|
1450
|
+
`orizu datasets append --dataset ${dataset.id} --file <remaining-file>.`
|
|
1451
|
+
)
|
|
1452
|
+
}
|
|
1453
|
+
}
|
|
1454
|
+
|
|
1455
|
+
if (!dataset) {
|
|
1456
|
+
throw new Error('Dataset file contains no rows')
|
|
1457
|
+
}
|
|
1458
|
+
|
|
1459
|
+
console.log(`Uploaded dataset ${dataset.name} (${dataset.id}) with ${dataset.rowCount} rows.`)
|
|
1460
|
+
if (dataset.url) {
|
|
1461
|
+
console.log(`View dataset: ${formatTerminalLink(dataset.url)}`)
|
|
1462
|
+
}
|
|
1463
|
+
}
|
|
1464
|
+
|
|
1465
|
+
async function uploadDataset() {
|
|
1466
|
+
const projectArg = getArg('--project')
|
|
1467
|
+
const fileArg = getArg('--file')
|
|
1468
|
+
const name = getArg('--name')
|
|
1469
|
+
|
|
1470
|
+
if (!fileArg) {
|
|
1471
|
+
throw new Error('Usage: orizu datasets upload --file <path> [--project <team/project>] [--name <name>]')
|
|
1472
|
+
}
|
|
1473
|
+
|
|
1474
|
+
const file = expandHomePath(fileArg)
|
|
1475
|
+
const project = await resolveProjectSlug(projectArg)
|
|
1476
|
+
const datasetName = name || basename(file)
|
|
1477
|
+
|
|
1478
|
+
if (extname(file).toLowerCase() === '.jsonl') {
|
|
1479
|
+
await uploadJsonlDatasetInChunks(file, project, datasetName)
|
|
1480
|
+
return
|
|
1481
|
+
}
|
|
1482
|
+
|
|
1483
|
+
const { rows, sourceType } = parseDatasetFile(file)
|
|
1484
|
+
const data = await createDatasetFromRows(project, datasetName, sourceType, rows)
|
|
1379
1485
|
|
|
1380
1486
|
console.log(`Uploaded dataset ${data.dataset.name} (${data.dataset.id}) with ${data.dataset.rowCount} rows.`)
|
|
1381
1487
|
if (data.dataset.url) {
|
|
@@ -1455,6 +1561,56 @@ async function appendChunk(
|
|
|
1455
1561
|
}>(response, 'Dataset append')
|
|
1456
1562
|
}
|
|
1457
1563
|
|
|
1564
|
+
async function appendJsonlDatasetRowsInChunks(datasetId: string, file: string) {
|
|
1565
|
+
let totalAppended = 0
|
|
1566
|
+
let lastResult: { dataset: { id: string; name: string; rowCount: number } } | null = null
|
|
1567
|
+
let chunkIndex = 0
|
|
1568
|
+
const chunks = streamJsonlRowChunks(file)[Symbol.asyncIterator]()
|
|
1569
|
+
|
|
1570
|
+
while (true) {
|
|
1571
|
+
let nextChunk: IteratorResult<Array<Record<string, unknown>>>
|
|
1572
|
+
try {
|
|
1573
|
+
nextChunk = await chunks.next()
|
|
1574
|
+
} catch (error) {
|
|
1575
|
+
const message = error instanceof Error ? error.message : String(error)
|
|
1576
|
+
throw new Error(
|
|
1577
|
+
`Append stopped while reading the next JSONL chunk: ${message}\n` +
|
|
1578
|
+
`${totalAppended} rows from ${chunkIndex} chunk(s) were already appended. ` +
|
|
1579
|
+
`Fix the file, remove the first ${totalAppended} rows, and re-run the command.`
|
|
1580
|
+
)
|
|
1581
|
+
}
|
|
1582
|
+
|
|
1583
|
+
if (nextChunk.done) {
|
|
1584
|
+
break
|
|
1585
|
+
}
|
|
1586
|
+
|
|
1587
|
+
const chunk = nextChunk.value
|
|
1588
|
+
chunkIndex += 1
|
|
1589
|
+
console.log(`Uploading chunk ${chunkIndex} (${chunk.length} rows)...`)
|
|
1590
|
+
|
|
1591
|
+
try {
|
|
1592
|
+
const data = await appendChunk(datasetId, chunk)
|
|
1593
|
+
totalAppended += data.appendedCount
|
|
1594
|
+
lastResult = data
|
|
1595
|
+
} catch (error) {
|
|
1596
|
+
const message = error instanceof Error ? error.message : String(error)
|
|
1597
|
+
throw new Error(
|
|
1598
|
+
`Chunk ${chunkIndex} failed: ${message}\n` +
|
|
1599
|
+
`${totalAppended} rows from ${chunkIndex - 1} chunk(s) were already appended. ` +
|
|
1600
|
+
`To retry, remove the first ${totalAppended} rows from your file and re-run the command.`
|
|
1601
|
+
)
|
|
1602
|
+
}
|
|
1603
|
+
}
|
|
1604
|
+
|
|
1605
|
+
if (!lastResult) {
|
|
1606
|
+
throw new Error('Dataset append file must contain at least one row')
|
|
1607
|
+
}
|
|
1608
|
+
|
|
1609
|
+
console.log(
|
|
1610
|
+
`Appended ${totalAppended} rows to dataset ${lastResult.dataset.name} (${lastResult.dataset.id}). New row count: ${lastResult.dataset.rowCount}`
|
|
1611
|
+
)
|
|
1612
|
+
}
|
|
1613
|
+
|
|
1458
1614
|
async function appendDatasetRows() {
|
|
1459
1615
|
const projectArg = getArg('--project')
|
|
1460
1616
|
const datasetInput = getDatasetReferenceInput()
|
|
@@ -1474,6 +1630,11 @@ async function appendDatasetRows() {
|
|
|
1474
1630
|
|
|
1475
1631
|
const file = expandHomePath(fileArg)
|
|
1476
1632
|
|
|
1633
|
+
if (extname(file).toLowerCase() === '.jsonl') {
|
|
1634
|
+
await appendJsonlDatasetRowsInChunks(datasetId, file)
|
|
1635
|
+
return
|
|
1636
|
+
}
|
|
1637
|
+
|
|
1477
1638
|
// Check file size before reading to prevent OOM on large files (ALI-565).
|
|
1478
1639
|
// Wrap statSync in try/catch so missing/inaccessible files get friendly
|
|
1479
1640
|
// errors instead of raw Node.js ENOENT/EPERM (ALI-554).
|
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
import { createReadStream } from 'fs'
|
|
2
|
+
import { createInterface } from 'readline'
|
|
3
|
+
|
|
4
|
+
export const JSONL_UPLOAD_CHUNK_MAX_BYTES = 2 * 1024 * 1024
|
|
5
|
+
|
|
6
|
+
interface JsonlChunkOptions {
|
|
7
|
+
maxPayloadBytes?: number
|
|
8
|
+
}
|
|
9
|
+
|
|
10
|
+
interface PendingRow {
|
|
11
|
+
row: Record<string, unknown>
|
|
12
|
+
jsonByteLength: number
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
function isObjectRow(value: unknown): value is Record<string, unknown> {
|
|
16
|
+
return typeof value === 'object' && value !== null && !Array.isArray(value)
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
function rowsPayloadByteLength(rowCount: number, rowsByteLength: number): number {
|
|
20
|
+
return Buffer.byteLength('{"rows":[]}', 'utf8') +
|
|
21
|
+
rowsByteLength +
|
|
22
|
+
Math.max(0, rowCount - 1)
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
function formatMb(byteLength: number): string {
|
|
26
|
+
return (byteLength / (1024 * 1024)).toFixed(1)
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
function mapReadError(filePath: string, error: unknown): Error {
|
|
30
|
+
const maybeError = error as NodeJS.ErrnoException
|
|
31
|
+
|
|
32
|
+
if (!maybeError.code && error instanceof Error) {
|
|
33
|
+
return error
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
if (maybeError.code === 'ENOENT') {
|
|
37
|
+
return new Error(
|
|
38
|
+
`File not found: ${filePath}. Check the path and filename, then retry.`
|
|
39
|
+
)
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
if (maybeError.code === 'EPERM' || maybeError.code === 'EACCES') {
|
|
43
|
+
return new Error(
|
|
44
|
+
`Cannot read file: ${filePath}. macOS may be blocking access to this folder (for example Downloads). Grant folder permission to your terminal app and retry.`
|
|
45
|
+
)
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
return new Error(`Failed to read file ${filePath}: ${maybeError.message}`)
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
export function estimateRowsPayloadByteLength(rows: Array<Record<string, unknown>>): number {
|
|
52
|
+
const rowsByteLength = rows.reduce((total, row) => {
|
|
53
|
+
return total + Buffer.byteLength(JSON.stringify(row), 'utf8')
|
|
54
|
+
}, 0)
|
|
55
|
+
|
|
56
|
+
return rowsPayloadByteLength(rows.length, rowsByteLength)
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
export async function* streamJsonlRowChunks(
|
|
60
|
+
filePath: string,
|
|
61
|
+
options: JsonlChunkOptions = {}
|
|
62
|
+
): AsyncGenerator<Array<Record<string, unknown>>> {
|
|
63
|
+
const maxPayloadBytes = options.maxPayloadBytes ?? JSONL_UPLOAD_CHUNK_MAX_BYTES
|
|
64
|
+
let chunk: PendingRow[] = []
|
|
65
|
+
let chunkRowsByteLength = 0
|
|
66
|
+
let parsedRowCount = 0
|
|
67
|
+
let lineNumber = 0
|
|
68
|
+
|
|
69
|
+
const stream = createReadStream(filePath, { encoding: 'utf8' })
|
|
70
|
+
const lines = createInterface({
|
|
71
|
+
input: stream,
|
|
72
|
+
crlfDelay: Infinity,
|
|
73
|
+
})
|
|
74
|
+
|
|
75
|
+
try {
|
|
76
|
+
for await (const line of lines) {
|
|
77
|
+
lineNumber += 1
|
|
78
|
+
const trimmed = line.trim()
|
|
79
|
+
if (!trimmed) {
|
|
80
|
+
continue
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
let parsed: unknown
|
|
84
|
+
try {
|
|
85
|
+
parsed = JSON.parse(trimmed)
|
|
86
|
+
} catch {
|
|
87
|
+
throw new Error(`Invalid JSONL at line ${lineNumber}`)
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
if (!isObjectRow(parsed)) {
|
|
91
|
+
throw new Error(`Row ${lineNumber} must be a JSON object`)
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
const jsonByteLength = Buffer.byteLength(JSON.stringify(parsed), 'utf8')
|
|
95
|
+
const singleRowPayloadBytes = rowsPayloadByteLength(1, jsonByteLength)
|
|
96
|
+
if (singleRowPayloadBytes > maxPayloadBytes) {
|
|
97
|
+
throw new Error(
|
|
98
|
+
`Row ${lineNumber} is ${formatMb(singleRowPayloadBytes)} MB after JSON encoding, which exceeds the ${formatMb(maxPayloadBytes)} MB upload chunk limit`
|
|
99
|
+
)
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
const nextPayloadBytes = rowsPayloadByteLength(
|
|
103
|
+
chunk.length + 1,
|
|
104
|
+
chunkRowsByteLength + jsonByteLength
|
|
105
|
+
)
|
|
106
|
+
|
|
107
|
+
if (chunk.length > 0 && nextPayloadBytes > maxPayloadBytes) {
|
|
108
|
+
yield chunk.map(item => item.row)
|
|
109
|
+
chunk = []
|
|
110
|
+
chunkRowsByteLength = 0
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
chunk.push({ row: parsed, jsonByteLength })
|
|
114
|
+
chunkRowsByteLength += jsonByteLength
|
|
115
|
+
parsedRowCount += 1
|
|
116
|
+
}
|
|
117
|
+
} catch (error) {
|
|
118
|
+
throw mapReadError(filePath, error)
|
|
119
|
+
} finally {
|
|
120
|
+
lines.close()
|
|
121
|
+
stream.destroy()
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
if (chunk.length > 0) {
|
|
125
|
+
yield chunk.map(item => item.row)
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
if (parsedRowCount === 0) {
|
|
129
|
+
throw new Error('Dataset file contains no rows')
|
|
130
|
+
}
|
|
131
|
+
}
|