orizu 0.1.0 → 0.1.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -1,6 +1,6 @@
1
1
  #!/usr/bin/env node
2
2
  import { createHash, randomBytes } from 'crypto';
3
- import { basename } from 'path';
3
+ import { basename, extname } from 'path';
4
4
  import { createServer } from 'http';
5
5
  import { readFileSync, statSync, writeFileSync } from 'fs';
6
6
  import { spawn } from 'child_process';
@@ -8,8 +8,8 @@ import { createInterface } from 'readline/promises';
8
8
  import { stdin as input, stdout as output } from 'process';
9
9
  import { clearServerCredentials, getServerCredentials, saveServerCredentials } from './credentials.js';
10
10
  import { parseDatasetFile } from './file-parser.js';
11
+ import { streamJsonlRowChunks } from './jsonl-stream.js';
11
12
  import { parseDatasetReference } from './dataset-download.js';
12
- import { extractErrorMessage } from './error-response.js';
13
13
  import { parseGlobalFlags } from './global-flags.js';
14
14
  import { authedFetch, getBaseUrl, resolveLoginBaseUrl, setGlobalFlags } from './http.js';
15
15
  import { formatTaskCreateError } from './task-create-error.js';
@@ -24,6 +24,9 @@ function getArg(name) {
24
24
  }
25
25
  return cliArgs[index + 1];
26
26
  }
27
+ function normalizeSlugInput(slug) {
28
+ return slug.trim().toLowerCase();
29
+ }
27
30
  function isInteractiveTerminal() {
28
31
  return Boolean(process.stdin.isTTY && process.stdout.isTTY);
29
32
  }
@@ -124,7 +127,8 @@ async function fetchTeams() {
124
127
  return data.teams;
125
128
  }
126
129
  async function fetchProjects(teamSlug) {
127
- const query = teamSlug ? `?teamSlug=${encodeURIComponent(teamSlug)}` : '';
130
+ const normalizedTeamSlug = teamSlug ? normalizeSlugInput(teamSlug) : undefined;
131
+ const query = normalizedTeamSlug ? `?teamSlug=${encodeURIComponent(normalizedTeamSlug)}` : '';
128
132
  const response = await authedFetch(`/api/cli/projects${query}`);
129
133
  if (!response.ok) {
130
134
  throw new Error(`Failed to fetch projects: ${await response.text()}`);
@@ -180,7 +184,7 @@ async function resolveProjectSlug(projectArg) {
180
184
  if (segments.length !== 2 || !segments[0] || !segments[1]) {
181
185
  throw new Error('Project must be in format teamSlug/projectSlug');
182
186
  }
183
- const [teamSlug, projectSlug] = segments;
187
+ const [teamSlug, projectSlug] = segments.map(normalizeSlugInput);
184
188
  const matchedTeam = teams.find(team => team.slug === teamSlug);
185
189
  if (!matchedTeam) {
186
190
  console.error(`Team '${teamSlug}' not found in your accessible teams.`);
@@ -189,14 +193,14 @@ async function resolveProjectSlug(projectArg) {
189
193
  const selectedProject = await promptSelect(`Select a project in ${selectedTeam.slug}`, projects, project => `${project.name} (${project.teamSlug}/${project.slug})`);
190
194
  return `${selectedProject.teamSlug}/${selectedProject.slug}`;
191
195
  }
192
- const projects = await fetchProjects(teamSlug);
196
+ const projects = await fetchProjects(matchedTeam.slug);
193
197
  const matchedProject = projects.find(project => project.slug === projectSlug);
194
198
  if (!matchedProject) {
195
- console.error(`Project '${projectSlug}' not found in team '${teamSlug}'.`);
196
- const selectedProject = await promptSelect(`Select a project in ${teamSlug}`, projects, project => `${project.name} (${project.teamSlug}/${project.slug})`);
199
+ console.error(`Project '${projectSlug}' not found in team '${matchedTeam.slug}'.`);
200
+ const selectedProject = await promptSelect(`Select a project in ${matchedTeam.slug}`, projects, project => `${project.name} (${project.teamSlug}/${project.slug})`);
197
201
  return `${selectedProject.teamSlug}/${selectedProject.slug}`;
198
202
  }
199
- return `${teamSlug}/${projectSlug}`;
203
+ return `${matchedTeam.slug}/${matchedProject.slug}`;
200
204
  }
201
205
  async function selectTaskIdInteractively() {
202
206
  const team = await promptSelect('Select a team', await fetchTeams(), item => `${item.name} (${item.slug})`, { forcePrompt: true });
@@ -460,7 +464,7 @@ async function listTeams() {
460
464
  }
461
465
  async function resolveTeamSlug(teamSlugArg) {
462
466
  if (teamSlugArg) {
463
- return teamSlugArg;
467
+ return normalizeSlugInput(teamSlugArg);
464
468
  }
465
469
  const team = await promptSelect('Select a team', await fetchTeams(), item => `${item.name} (${item.slug})`, { forcePrompt: true });
466
470
  return team.slug;
@@ -491,12 +495,14 @@ async function createTeam() {
491
495
  console.log(`Created team: ${data.team.name} (${data.team.slug})`);
492
496
  }
493
497
  async function listProjects() {
494
- const teamSlug = getArg('--team');
498
+ const teamSlugArg = getArg('--team');
499
+ const teamSlug = teamSlugArg ? normalizeSlugInput(teamSlugArg) : null;
495
500
  printProjects(await fetchProjects(teamSlug || undefined));
496
501
  }
497
502
  async function createProject() {
498
503
  const name = getArg('--name');
499
- let teamSlug = getArg('--team');
504
+ const teamSlugArg = getArg('--team');
505
+ let teamSlug = teamSlugArg ? normalizeSlugInput(teamSlugArg) : null;
500
506
  if (!name) {
501
507
  throw new Error('Usage: orizu projects create --name <name> [--team <teamSlug>]');
502
508
  }
@@ -700,7 +706,6 @@ async function createTask() {
700
706
  ...payload,
701
707
  httpStatus: cliError.httpStatus,
702
708
  }, null, 2));
703
- process.exit(1);
704
709
  }
705
710
  throw cliError;
706
711
  }
@@ -756,8 +761,8 @@ async function taskStatus() {
756
761
  }
757
762
  const response = await authedFetch(`/api/cli/tasks/${encodeURIComponent(taskId)}/status`);
758
763
  if (!response.ok) {
764
+ const rawBody = await response.text();
759
765
  if (hasArg('--json')) {
760
- const rawBody = await response.text();
761
766
  let errorPayload = { error: rawBody };
762
767
  try {
763
768
  const parsed = JSON.parse(rawBody);
@@ -772,9 +777,16 @@ async function taskStatus() {
772
777
  ...errorPayload,
773
778
  httpStatus: response.status,
774
779
  }, null, 2));
775
- process.exit(1);
776
780
  }
777
- throw new Error(`Failed to fetch task status: ${await extractErrorMessage(response)}`);
781
+ let errorMsg = rawBody;
782
+ try {
783
+ const parsed = JSON.parse(rawBody);
784
+ if (parsed && typeof parsed === 'object' && typeof parsed.error === 'string') {
785
+ errorMsg = parsed.error;
786
+ }
787
+ }
788
+ catch { /* use rawBody as-is */ }
789
+ throw new Error(`Failed to fetch task status: ${errorMsg}`);
778
790
  }
779
791
  const data = await parseJsonResponse(response, 'Task status');
780
792
  if (hasArg('--json')) {
@@ -809,12 +821,7 @@ async function appDetail() {
809
821
  throw new Error('Usage: orizu apps detail --app <appId> [--project <team/project>] [--json]');
810
822
  }
811
823
  const projectSlug = project || await resolveProjectSlug(null);
812
- const apps = await fetchApps(projectSlug);
813
- const app = apps.find(a => a.id === appId);
814
- if (!app) {
815
- throw new Error(`App '${appId}' not found in project '${projectSlug}'`);
816
- }
817
- // Re-fetch full detail from the list (it already returns full data)
824
+ // Single fetch — the apps endpoint already returns full detail (ALI-544)
818
825
  const detailResponse = await authedFetch(`/api/cli/apps?project=${encodeURIComponent(projectSlug)}`);
819
826
  if (!detailResponse.ok) {
820
827
  throw new Error(`Failed to fetch app detail: ${await detailResponse.text()}`);
@@ -822,7 +829,7 @@ async function appDetail() {
822
829
  const detailData = await parseJsonResponse(detailResponse, 'App detail');
823
830
  const detail = detailData.apps.find(a => a.id === appId);
824
831
  if (!detail) {
825
- throw new Error(`App '${appId}' not found`);
832
+ throw new Error(`App '${appId}' not found in project '${projectSlug}'`);
826
833
  }
827
834
  if (hasArg('--json')) {
828
835
  console.log(JSON.stringify(detail, null, 2));
@@ -884,7 +891,8 @@ async function removeTeamMember() {
884
891
  console.log(`Removed team member ${member.email}`);
885
892
  }
886
893
  async function changeTeamMemberRole() {
887
- const teamSlug = getArg('--team');
894
+ const teamSlugArg = getArg('--team');
895
+ const teamSlug = teamSlugArg ? normalizeSlugInput(teamSlugArg) : null;
888
896
  const email = getArg('--email');
889
897
  const role = getArg('--role');
890
898
  if (!teamSlug || !email || !role) {
@@ -908,17 +916,7 @@ async function changeTeamMemberRole() {
908
916
  }
909
917
  console.log(`Updated ${member.email} role to ${role}`);
910
918
  }
911
- async function uploadDataset() {
912
- const projectArg = getArg('--project');
913
- const fileArg = getArg('--file');
914
- const name = getArg('--name');
915
- if (!fileArg) {
916
- throw new Error('Usage: orizu datasets upload --file <path> [--project <team/project>] [--name <name>]');
917
- }
918
- const file = expandHomePath(fileArg);
919
- const project = await resolveProjectSlug(projectArg);
920
- const { rows, sourceType } = parseDatasetFile(file);
921
- const datasetName = name || basename(file);
919
+ async function createDatasetFromRows(project, name, sourceType, rows) {
922
920
  const response = await authedFetch('/api/cli/datasets/upload', {
923
921
  method: 'POST',
924
922
  headers: {
@@ -926,7 +924,7 @@ async function uploadDataset() {
926
924
  },
927
925
  body: JSON.stringify({
928
926
  projectSlug: project,
929
- name: datasetName,
927
+ name,
930
928
  rows,
931
929
  sourceType,
932
930
  }),
@@ -935,7 +933,83 @@ async function uploadDataset() {
935
933
  const body = await parseJsonResponse(response, 'Dataset upload');
936
934
  throw new Error(`Upload failed: ${body.error}`);
937
935
  }
938
- const data = await parseJsonResponse(response, 'Dataset upload');
936
+ return parseJsonResponse(response, 'Dataset upload');
937
+ }
938
+ async function uploadJsonlDatasetInChunks(file, project, datasetName) {
939
+ let dataset = null;
940
+ let totalUploaded = 0;
941
+ let chunkIndex = 0;
942
+ const chunks = streamJsonlRowChunks(file)[Symbol.asyncIterator]();
943
+ while (true) {
944
+ let nextChunk;
945
+ try {
946
+ nextChunk = await chunks.next();
947
+ }
948
+ catch (error) {
949
+ const message = error instanceof Error ? error.message : String(error);
950
+ if (!dataset) {
951
+ throw new Error(message);
952
+ }
953
+ throw new Error(`Upload stopped while reading the next JSONL chunk: ${message}\n` +
954
+ `Dataset ${dataset.name} (${dataset.id}) was created and ${totalUploaded} rows were uploaded. ` +
955
+ `Fix the file, remove the first ${totalUploaded} rows, and run ` +
956
+ `orizu datasets append --dataset ${dataset.id} --file <remaining-file>.`);
957
+ }
958
+ if (nextChunk.done) {
959
+ break;
960
+ }
961
+ const chunk = nextChunk.value;
962
+ chunkIndex += 1;
963
+ console.log(`Uploading chunk ${chunkIndex} (${chunk.length} rows)...`);
964
+ try {
965
+ if (!dataset) {
966
+ const data = await createDatasetFromRows(project, datasetName, 'jsonl', chunk);
967
+ dataset = data.dataset;
968
+ totalUploaded = data.dataset.rowCount;
969
+ continue;
970
+ }
971
+ const data = await appendChunk(dataset.id, chunk);
972
+ totalUploaded += data.appendedCount;
973
+ dataset = {
974
+ ...dataset,
975
+ rowCount: data.dataset.rowCount,
976
+ };
977
+ }
978
+ catch (error) {
979
+ const message = error instanceof Error ? error.message : String(error);
980
+ if (!dataset) {
981
+ throw new Error(message);
982
+ }
983
+ throw new Error(`Chunk ${chunkIndex} failed: ${message}\n` +
984
+ `Dataset ${dataset.name} (${dataset.id}) was created and ${totalUploaded} rows were uploaded. ` +
985
+ `To retry, remove the first ${totalUploaded} rows from your file and run ` +
986
+ `orizu datasets append --dataset ${dataset.id} --file <remaining-file>.`);
987
+ }
988
+ }
989
+ if (!dataset) {
990
+ throw new Error('Dataset file contains no rows');
991
+ }
992
+ console.log(`Uploaded dataset ${dataset.name} (${dataset.id}) with ${dataset.rowCount} rows.`);
993
+ if (dataset.url) {
994
+ console.log(`View dataset: ${formatTerminalLink(dataset.url)}`);
995
+ }
996
+ }
997
+ async function uploadDataset() {
998
+ const projectArg = getArg('--project');
999
+ const fileArg = getArg('--file');
1000
+ const name = getArg('--name');
1001
+ if (!fileArg) {
1002
+ throw new Error('Usage: orizu datasets upload --file <path> [--project <team/project>] [--name <name>]');
1003
+ }
1004
+ const file = expandHomePath(fileArg);
1005
+ const project = await resolveProjectSlug(projectArg);
1006
+ const datasetName = name || basename(file);
1007
+ if (extname(file).toLowerCase() === '.jsonl') {
1008
+ await uploadJsonlDatasetInChunks(file, project, datasetName);
1009
+ return;
1010
+ }
1011
+ const { rows, sourceType } = parseDatasetFile(file);
1012
+ const data = await createDatasetFromRows(project, datasetName, sourceType, rows);
939
1013
  console.log(`Uploaded dataset ${data.dataset.name} (${data.dataset.id}) with ${data.dataset.rowCount} rows.`);
940
1014
  if (data.dataset.url) {
941
1015
  console.log(`View dataset: ${formatTerminalLink(data.dataset.url)}`);
@@ -992,6 +1066,45 @@ async function appendChunk(datasetId, rows) {
992
1066
  }
993
1067
  return parseJsonResponse(response, 'Dataset append');
994
1068
  }
1069
+ async function appendJsonlDatasetRowsInChunks(datasetId, file) {
1070
+ let totalAppended = 0;
1071
+ let lastResult = null;
1072
+ let chunkIndex = 0;
1073
+ const chunks = streamJsonlRowChunks(file)[Symbol.asyncIterator]();
1074
+ while (true) {
1075
+ let nextChunk;
1076
+ try {
1077
+ nextChunk = await chunks.next();
1078
+ }
1079
+ catch (error) {
1080
+ const message = error instanceof Error ? error.message : String(error);
1081
+ throw new Error(`Append stopped while reading the next JSONL chunk: ${message}\n` +
1082
+ `${totalAppended} rows from ${chunkIndex} chunk(s) were already appended. ` +
1083
+ `Fix the file, remove the first ${totalAppended} rows, and re-run the command.`);
1084
+ }
1085
+ if (nextChunk.done) {
1086
+ break;
1087
+ }
1088
+ const chunk = nextChunk.value;
1089
+ chunkIndex += 1;
1090
+ console.log(`Uploading chunk ${chunkIndex} (${chunk.length} rows)...`);
1091
+ try {
1092
+ const data = await appendChunk(datasetId, chunk);
1093
+ totalAppended += data.appendedCount;
1094
+ lastResult = data;
1095
+ }
1096
+ catch (error) {
1097
+ const message = error instanceof Error ? error.message : String(error);
1098
+ throw new Error(`Chunk ${chunkIndex} failed: ${message}\n` +
1099
+ `${totalAppended} rows from ${chunkIndex - 1} chunk(s) were already appended. ` +
1100
+ `To retry, remove the first ${totalAppended} rows from your file and re-run the command.`);
1101
+ }
1102
+ }
1103
+ if (!lastResult) {
1104
+ throw new Error('Dataset append file must contain at least one row');
1105
+ }
1106
+ console.log(`Appended ${totalAppended} rows to dataset ${lastResult.dataset.name} (${lastResult.dataset.id}). New row count: ${lastResult.dataset.rowCount}`);
1107
+ }
995
1108
  async function appendDatasetRows() {
996
1109
  const projectArg = getArg('--project');
997
1110
  const datasetInput = getDatasetReferenceInput();
@@ -1008,18 +1121,35 @@ async function appendDatasetRows() {
1008
1121
  datasetId = selected.datasetId;
1009
1122
  }
1010
1123
  const file = expandHomePath(fileArg);
1011
- // Parse the file first — parseDatasetFile provides user-friendly errors for
1012
- // ENOENT, EPERM, and EACCES. We check the file size afterward using the
1013
- // already-read content length to avoid a raw statSync ENOENT (ALI-554).
1014
- const { rows } = parseDatasetFile(file);
1015
- if (!Array.isArray(rows) || rows.length === 0) {
1016
- throw new Error('Dataset append file must contain at least one row');
1124
+ if (extname(file).toLowerCase() === '.jsonl') {
1125
+ await appendJsonlDatasetRowsInChunks(datasetId, file);
1126
+ return;
1127
+ }
1128
+ // Check file size before reading to prevent OOM on large files (ALI-565).
1129
+ // Wrap statSync in try/catch so missing/inaccessible files get friendly
1130
+ // errors instead of raw Node.js ENOENT/EPERM (ALI-554).
1131
+ let fileSizeBytes;
1132
+ try {
1133
+ fileSizeBytes = statSync(file).size;
1134
+ }
1135
+ catch (error) {
1136
+ const maybeError = error;
1137
+ if (maybeError.code === 'ENOENT') {
1138
+ throw new Error(`File not found: ${file}. Check the path and filename, then retry.`);
1139
+ }
1140
+ if (maybeError.code === 'EPERM' || maybeError.code === 'EACCES') {
1141
+ throw new Error(`Cannot read file: ${file}. Grant folder permission to your terminal app and retry.`);
1142
+ }
1143
+ throw new Error(`Failed to access file ${file}: ${maybeError.message}`);
1017
1144
  }
1018
- const fileSizeBytes = statSync(file).size;
1019
1145
  if (fileSizeBytes > MAX_INPUT_FILE_SIZE_BYTES) {
1020
1146
  const sizeMb = (fileSizeBytes / (1024 * 1024)).toFixed(1);
1021
1147
  throw new Error(`Input file is ${sizeMb} MB, which exceeds the 50 MB limit. Split the file into smaller parts and append each separately.`);
1022
1148
  }
1149
+ const { rows } = parseDatasetFile(file);
1150
+ if (!Array.isArray(rows) || rows.length === 0) {
1151
+ throw new Error('Dataset append file must contain at least one row');
1152
+ }
1023
1153
  if (rows.length <= APPEND_CHUNK_SIZE_ROWS) {
1024
1154
  const data = await appendChunk(datasetId, rows);
1025
1155
  console.log(`Appended ${data.appendedCount} rows to dataset ${data.dataset.name} (${data.dataset.id}). New row count: ${data.dataset.rowCount}`);
@@ -0,0 +1,91 @@
1
+ import { createReadStream } from 'fs';
2
+ import { createInterface } from 'readline';
3
+ export const JSONL_UPLOAD_CHUNK_MAX_BYTES = 2 * 1024 * 1024;
4
+ function isObjectRow(value) {
5
+ return typeof value === 'object' && value !== null && !Array.isArray(value);
6
+ }
7
+ function rowsPayloadByteLength(rowCount, rowsByteLength) {
8
+ return Buffer.byteLength('{"rows":[]}', 'utf8') +
9
+ rowsByteLength +
10
+ Math.max(0, rowCount - 1);
11
+ }
12
+ function formatMb(byteLength) {
13
+ return (byteLength / (1024 * 1024)).toFixed(1);
14
+ }
15
+ function mapReadError(filePath, error) {
16
+ const maybeError = error;
17
+ if (!maybeError.code && error instanceof Error) {
18
+ return error;
19
+ }
20
+ if (maybeError.code === 'ENOENT') {
21
+ return new Error(`File not found: ${filePath}. Check the path and filename, then retry.`);
22
+ }
23
+ if (maybeError.code === 'EPERM' || maybeError.code === 'EACCES') {
24
+ return new Error(`Cannot read file: ${filePath}. macOS may be blocking access to this folder (for example Downloads). Grant folder permission to your terminal app and retry.`);
25
+ }
26
+ return new Error(`Failed to read file ${filePath}: ${maybeError.message}`);
27
+ }
28
+ export function estimateRowsPayloadByteLength(rows) {
29
+ const rowsByteLength = rows.reduce((total, row) => {
30
+ return total + Buffer.byteLength(JSON.stringify(row), 'utf8');
31
+ }, 0);
32
+ return rowsPayloadByteLength(rows.length, rowsByteLength);
33
+ }
34
+ export async function* streamJsonlRowChunks(filePath, options = {}) {
35
+ const maxPayloadBytes = options.maxPayloadBytes ?? JSONL_UPLOAD_CHUNK_MAX_BYTES;
36
+ let chunk = [];
37
+ let chunkRowsByteLength = 0;
38
+ let parsedRowCount = 0;
39
+ let lineNumber = 0;
40
+ const stream = createReadStream(filePath, { encoding: 'utf8' });
41
+ const lines = createInterface({
42
+ input: stream,
43
+ crlfDelay: Infinity,
44
+ });
45
+ try {
46
+ for await (const line of lines) {
47
+ lineNumber += 1;
48
+ const trimmed = line.trim();
49
+ if (!trimmed) {
50
+ continue;
51
+ }
52
+ let parsed;
53
+ try {
54
+ parsed = JSON.parse(trimmed);
55
+ }
56
+ catch {
57
+ throw new Error(`Invalid JSONL at line ${lineNumber}`);
58
+ }
59
+ if (!isObjectRow(parsed)) {
60
+ throw new Error(`Row ${lineNumber} must be a JSON object`);
61
+ }
62
+ const jsonByteLength = Buffer.byteLength(JSON.stringify(parsed), 'utf8');
63
+ const singleRowPayloadBytes = rowsPayloadByteLength(1, jsonByteLength);
64
+ if (singleRowPayloadBytes > maxPayloadBytes) {
65
+ throw new Error(`Row ${lineNumber} is ${formatMb(singleRowPayloadBytes)} MB after JSON encoding, which exceeds the ${formatMb(maxPayloadBytes)} MB upload chunk limit`);
66
+ }
67
+ const nextPayloadBytes = rowsPayloadByteLength(chunk.length + 1, chunkRowsByteLength + jsonByteLength);
68
+ if (chunk.length > 0 && nextPayloadBytes > maxPayloadBytes) {
69
+ yield chunk.map(item => item.row);
70
+ chunk = [];
71
+ chunkRowsByteLength = 0;
72
+ }
73
+ chunk.push({ row: parsed, jsonByteLength });
74
+ chunkRowsByteLength += jsonByteLength;
75
+ parsedRowCount += 1;
76
+ }
77
+ }
78
+ catch (error) {
79
+ throw mapReadError(filePath, error);
80
+ }
81
+ finally {
82
+ lines.close();
83
+ stream.destroy();
84
+ }
85
+ if (chunk.length > 0) {
86
+ yield chunk.map(item => item.row);
87
+ }
88
+ if (parsedRowCount === 0) {
89
+ throw new Error('Dataset file contains no rows');
90
+ }
91
+ }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "orizu",
3
- "version": "0.1.0",
3
+ "version": "0.1.2",
4
4
  "private": false,
5
5
  "type": "module",
6
6
  "bin": {
package/src/index.ts CHANGED
@@ -1,6 +1,6 @@
1
1
  #!/usr/bin/env node
2
2
  import { createHash, randomBytes } from 'crypto'
3
- import { basename } from 'path'
3
+ import { basename, extname } from 'path'
4
4
  import { createServer } from 'http'
5
5
  import { readFileSync, statSync, writeFileSync } from 'fs'
6
6
  import { spawn } from 'child_process'
@@ -8,8 +8,8 @@ import { createInterface } from 'readline/promises'
8
8
  import { stdin as input, stdout as output } from 'process'
9
9
  import { clearServerCredentials, getServerCredentials, saveServerCredentials } from './credentials.js'
10
10
  import { parseDatasetFile } from './file-parser.js'
11
+ import { streamJsonlRowChunks } from './jsonl-stream.js'
11
12
  import { parseDatasetReference } from './dataset-download.js'
12
- import { extractErrorMessage } from './error-response.js'
13
13
  import { parseGlobalFlags } from './global-flags.js'
14
14
  import { authedFetch, getBaseUrl, resolveLoginBaseUrl, setGlobalFlags } from './http.js'
15
15
  import { formatTaskCreateError } from './task-create-error.js'
@@ -67,6 +67,18 @@ interface DatasetSummary {
67
67
  teamSlug: string
68
68
  }
69
69
 
70
+ type DatasetUploadSourceType = 'csv' | 'json' | 'jsonl'
71
+
72
+ interface DatasetUploadResponse {
73
+ dataset: {
74
+ id: string
75
+ name: string
76
+ rowCount: number
77
+ sourceType: string
78
+ url?: string
79
+ }
80
+ }
81
+
70
82
  interface TeamMember {
71
83
  id: string
72
84
  user_id: string | null
@@ -122,6 +134,10 @@ function getArg(name: string): string | null {
122
134
  return cliArgs[index + 1]
123
135
  }
124
136
 
137
+ function normalizeSlugInput(slug: string): string {
138
+ return slug.trim().toLowerCase()
139
+ }
140
+
125
141
  function isInteractiveTerminal() {
126
142
  return Boolean(process.stdin.isTTY && process.stdout.isTTY)
127
143
  }
@@ -252,7 +268,8 @@ async function fetchTeams(): Promise<Team[]> {
252
268
  }
253
269
 
254
270
  async function fetchProjects(teamSlug?: string): Promise<Project[]> {
255
- const query = teamSlug ? `?teamSlug=${encodeURIComponent(teamSlug)}` : ''
271
+ const normalizedTeamSlug = teamSlug ? normalizeSlugInput(teamSlug) : undefined
272
+ const query = normalizedTeamSlug ? `?teamSlug=${encodeURIComponent(normalizedTeamSlug)}` : ''
256
273
  const response = await authedFetch(`/api/cli/projects${query}`)
257
274
  if (!response.ok) {
258
275
  throw new Error(`Failed to fetch projects: ${await response.text()}`)
@@ -333,7 +350,7 @@ async function resolveProjectSlug(projectArg: string | null): Promise<string> {
333
350
  if (segments.length !== 2 || !segments[0] || !segments[1]) {
334
351
  throw new Error('Project must be in format teamSlug/projectSlug')
335
352
  }
336
- const [teamSlug, projectSlug] = segments
353
+ const [teamSlug, projectSlug] = segments.map(normalizeSlugInput)
337
354
 
338
355
  const matchedTeam = teams.find(team => team.slug === teamSlug)
339
356
  if (!matchedTeam) {
@@ -354,13 +371,13 @@ async function resolveProjectSlug(projectArg: string | null): Promise<string> {
354
371
  return `${selectedProject.teamSlug}/${selectedProject.slug}`
355
372
  }
356
373
 
357
- const projects = await fetchProjects(teamSlug)
374
+ const projects = await fetchProjects(matchedTeam.slug)
358
375
  const matchedProject = projects.find(project => project.slug === projectSlug)
359
376
 
360
377
  if (!matchedProject) {
361
- console.error(`Project '${projectSlug}' not found in team '${teamSlug}'.`)
378
+ console.error(`Project '${projectSlug}' not found in team '${matchedTeam.slug}'.`)
362
379
  const selectedProject = await promptSelect(
363
- `Select a project in ${teamSlug}`,
380
+ `Select a project in ${matchedTeam.slug}`,
364
381
  projects,
365
382
  project => `${project.name} (${project.teamSlug}/${project.slug})`
366
383
  )
@@ -368,7 +385,7 @@ async function resolveProjectSlug(projectArg: string | null): Promise<string> {
368
385
  return `${selectedProject.teamSlug}/${selectedProject.slug}`
369
386
  }
370
387
 
371
- return `${teamSlug}/${projectSlug}`
388
+ return `${matchedTeam.slug}/${matchedProject.slug}`
372
389
  }
373
390
 
374
391
  async function selectTaskIdInteractively(): Promise<string> {
@@ -745,7 +762,7 @@ async function listTeams() {
745
762
 
746
763
  async function resolveTeamSlug(teamSlugArg: string | null): Promise<string> {
747
764
  if (teamSlugArg) {
748
- return teamSlugArg
765
+ return normalizeSlugInput(teamSlugArg)
749
766
  }
750
767
 
751
768
  const team = await promptSelect(
@@ -789,13 +806,15 @@ async function createTeam() {
789
806
  }
790
807
 
791
808
  async function listProjects() {
792
- const teamSlug = getArg('--team')
809
+ const teamSlugArg = getArg('--team')
810
+ const teamSlug = teamSlugArg ? normalizeSlugInput(teamSlugArg) : null
793
811
  printProjects(await fetchProjects(teamSlug || undefined))
794
812
  }
795
813
 
796
814
  async function createProject() {
797
815
  const name = getArg('--name')
798
- let teamSlug = getArg('--team')
816
+ const teamSlugArg = getArg('--team')
817
+ let teamSlug = teamSlugArg ? normalizeSlugInput(teamSlugArg) : null
799
818
 
800
819
  if (!name) {
801
820
  throw new Error('Usage: orizu projects create --name <name> [--team <teamSlug>]')
@@ -1308,7 +1327,8 @@ async function removeTeamMember() {
1308
1327
  }
1309
1328
 
1310
1329
  async function changeTeamMemberRole() {
1311
- const teamSlug = getArg('--team')
1330
+ const teamSlugArg = getArg('--team')
1331
+ const teamSlug = teamSlugArg ? normalizeSlugInput(teamSlugArg) : null
1312
1332
  const email = getArg('--email')
1313
1333
  const role = getArg('--role')
1314
1334
  if (!teamSlug || !email || !role) {
@@ -1340,21 +1360,12 @@ async function changeTeamMemberRole() {
1340
1360
  console.log(`Updated ${member.email} role to ${role}`)
1341
1361
  }
1342
1362
 
1343
- async function uploadDataset() {
1344
- const projectArg = getArg('--project')
1345
- const fileArg = getArg('--file')
1346
- const name = getArg('--name')
1347
-
1348
- if (!fileArg) {
1349
- throw new Error('Usage: orizu datasets upload --file <path> [--project <team/project>] [--name <name>]')
1350
- }
1351
-
1352
- const file = expandHomePath(fileArg)
1353
- const project = await resolveProjectSlug(projectArg)
1354
-
1355
- const { rows, sourceType } = parseDatasetFile(file)
1356
- const datasetName = name || basename(file)
1357
-
1363
+ async function createDatasetFromRows(
1364
+ project: string,
1365
+ name: string,
1366
+ sourceType: DatasetUploadSourceType,
1367
+ rows: Array<Record<string, unknown>>
1368
+ ): Promise<DatasetUploadResponse> {
1358
1369
  const response = await authedFetch('/api/cli/datasets/upload', {
1359
1370
  method: 'POST',
1360
1371
  headers: {
@@ -1362,7 +1373,7 @@ async function uploadDataset() {
1362
1373
  },
1363
1374
  body: JSON.stringify({
1364
1375
  projectSlug: project,
1365
- name: datasetName,
1376
+ name,
1366
1377
  rows,
1367
1378
  sourceType,
1368
1379
  }),
@@ -1373,9 +1384,104 @@ async function uploadDataset() {
1373
1384
  throw new Error(`Upload failed: ${body.error}`)
1374
1385
  }
1375
1386
 
1376
- const data = await parseJsonResponse<{
1377
- dataset: { id: string; name: string; rowCount: number; sourceType: string; url?: string }
1378
- }>(response, 'Dataset upload')
1387
+ return parseJsonResponse<DatasetUploadResponse>(response, 'Dataset upload')
1388
+ }
1389
+
1390
+ async function uploadJsonlDatasetInChunks(
1391
+ file: string,
1392
+ project: string,
1393
+ datasetName: string
1394
+ ) {
1395
+ let dataset: DatasetUploadResponse['dataset'] | null = null
1396
+ let totalUploaded = 0
1397
+ let chunkIndex = 0
1398
+ const chunks = streamJsonlRowChunks(file)[Symbol.asyncIterator]()
1399
+
1400
+ while (true) {
1401
+ let nextChunk: IteratorResult<Array<Record<string, unknown>>>
1402
+ try {
1403
+ nextChunk = await chunks.next()
1404
+ } catch (error) {
1405
+ const message = error instanceof Error ? error.message : String(error)
1406
+ if (!dataset) {
1407
+ throw new Error(message)
1408
+ }
1409
+
1410
+ throw new Error(
1411
+ `Upload stopped while reading the next JSONL chunk: ${message}\n` +
1412
+ `Dataset ${dataset.name} (${dataset.id}) was created and ${totalUploaded} rows were uploaded. ` +
1413
+ `Fix the file, remove the first ${totalUploaded} rows, and run ` +
1414
+ `orizu datasets append --dataset ${dataset.id} --file <remaining-file>.`
1415
+ )
1416
+ }
1417
+
1418
+ if (nextChunk.done) {
1419
+ break
1420
+ }
1421
+
1422
+ const chunk = nextChunk.value
1423
+ chunkIndex += 1
1424
+ console.log(`Uploading chunk ${chunkIndex} (${chunk.length} rows)...`)
1425
+
1426
+ try {
1427
+ if (!dataset) {
1428
+ const data = await createDatasetFromRows(project, datasetName, 'jsonl', chunk)
1429
+ dataset = data.dataset
1430
+ totalUploaded = data.dataset.rowCount
1431
+ continue
1432
+ }
1433
+
1434
+ const data = await appendChunk(dataset.id, chunk)
1435
+ totalUploaded += data.appendedCount
1436
+ dataset = {
1437
+ ...dataset,
1438
+ rowCount: data.dataset.rowCount,
1439
+ }
1440
+ } catch (error) {
1441
+ const message = error instanceof Error ? error.message : String(error)
1442
+ if (!dataset) {
1443
+ throw new Error(message)
1444
+ }
1445
+
1446
+ throw new Error(
1447
+ `Chunk ${chunkIndex} failed: ${message}\n` +
1448
+ `Dataset ${dataset.name} (${dataset.id}) was created and ${totalUploaded} rows were uploaded. ` +
1449
+ `To retry, remove the first ${totalUploaded} rows from your file and run ` +
1450
+ `orizu datasets append --dataset ${dataset.id} --file <remaining-file>.`
1451
+ )
1452
+ }
1453
+ }
1454
+
1455
+ if (!dataset) {
1456
+ throw new Error('Dataset file contains no rows')
1457
+ }
1458
+
1459
+ console.log(`Uploaded dataset ${dataset.name} (${dataset.id}) with ${dataset.rowCount} rows.`)
1460
+ if (dataset.url) {
1461
+ console.log(`View dataset: ${formatTerminalLink(dataset.url)}`)
1462
+ }
1463
+ }
1464
+
1465
+ async function uploadDataset() {
1466
+ const projectArg = getArg('--project')
1467
+ const fileArg = getArg('--file')
1468
+ const name = getArg('--name')
1469
+
1470
+ if (!fileArg) {
1471
+ throw new Error('Usage: orizu datasets upload --file <path> [--project <team/project>] [--name <name>]')
1472
+ }
1473
+
1474
+ const file = expandHomePath(fileArg)
1475
+ const project = await resolveProjectSlug(projectArg)
1476
+ const datasetName = name || basename(file)
1477
+
1478
+ if (extname(file).toLowerCase() === '.jsonl') {
1479
+ await uploadJsonlDatasetInChunks(file, project, datasetName)
1480
+ return
1481
+ }
1482
+
1483
+ const { rows, sourceType } = parseDatasetFile(file)
1484
+ const data = await createDatasetFromRows(project, datasetName, sourceType, rows)
1379
1485
 
1380
1486
  console.log(`Uploaded dataset ${data.dataset.name} (${data.dataset.id}) with ${data.dataset.rowCount} rows.`)
1381
1487
  if (data.dataset.url) {
@@ -1455,6 +1561,56 @@ async function appendChunk(
1455
1561
  }>(response, 'Dataset append')
1456
1562
  }
1457
1563
 
1564
+ async function appendJsonlDatasetRowsInChunks(datasetId: string, file: string) {
1565
+ let totalAppended = 0
1566
+ let lastResult: { dataset: { id: string; name: string; rowCount: number } } | null = null
1567
+ let chunkIndex = 0
1568
+ const chunks = streamJsonlRowChunks(file)[Symbol.asyncIterator]()
1569
+
1570
+ while (true) {
1571
+ let nextChunk: IteratorResult<Array<Record<string, unknown>>>
1572
+ try {
1573
+ nextChunk = await chunks.next()
1574
+ } catch (error) {
1575
+ const message = error instanceof Error ? error.message : String(error)
1576
+ throw new Error(
1577
+ `Append stopped while reading the next JSONL chunk: ${message}\n` +
1578
+ `${totalAppended} rows from ${chunkIndex} chunk(s) were already appended. ` +
1579
+ `Fix the file, remove the first ${totalAppended} rows, and re-run the command.`
1580
+ )
1581
+ }
1582
+
1583
+ if (nextChunk.done) {
1584
+ break
1585
+ }
1586
+
1587
+ const chunk = nextChunk.value
1588
+ chunkIndex += 1
1589
+ console.log(`Uploading chunk ${chunkIndex} (${chunk.length} rows)...`)
1590
+
1591
+ try {
1592
+ const data = await appendChunk(datasetId, chunk)
1593
+ totalAppended += data.appendedCount
1594
+ lastResult = data
1595
+ } catch (error) {
1596
+ const message = error instanceof Error ? error.message : String(error)
1597
+ throw new Error(
1598
+ `Chunk ${chunkIndex} failed: ${message}\n` +
1599
+ `${totalAppended} rows from ${chunkIndex - 1} chunk(s) were already appended. ` +
1600
+ `To retry, remove the first ${totalAppended} rows from your file and re-run the command.`
1601
+ )
1602
+ }
1603
+ }
1604
+
1605
+ if (!lastResult) {
1606
+ throw new Error('Dataset append file must contain at least one row')
1607
+ }
1608
+
1609
+ console.log(
1610
+ `Appended ${totalAppended} rows to dataset ${lastResult.dataset.name} (${lastResult.dataset.id}). New row count: ${lastResult.dataset.rowCount}`
1611
+ )
1612
+ }
1613
+
1458
1614
  async function appendDatasetRows() {
1459
1615
  const projectArg = getArg('--project')
1460
1616
  const datasetInput = getDatasetReferenceInput()
@@ -1474,6 +1630,11 @@ async function appendDatasetRows() {
1474
1630
 
1475
1631
  const file = expandHomePath(fileArg)
1476
1632
 
1633
+ if (extname(file).toLowerCase() === '.jsonl') {
1634
+ await appendJsonlDatasetRowsInChunks(datasetId, file)
1635
+ return
1636
+ }
1637
+
1477
1638
  // Check file size before reading to prevent OOM on large files (ALI-565).
1478
1639
  // Wrap statSync in try/catch so missing/inaccessible files get friendly
1479
1640
  // errors instead of raw Node.js ENOENT/EPERM (ALI-554).
@@ -0,0 +1,131 @@
1
+ import { createReadStream } from 'fs'
2
+ import { createInterface } from 'readline'
3
+
4
+ export const JSONL_UPLOAD_CHUNK_MAX_BYTES = 2 * 1024 * 1024
5
+
6
+ interface JsonlChunkOptions {
7
+ maxPayloadBytes?: number
8
+ }
9
+
10
+ interface PendingRow {
11
+ row: Record<string, unknown>
12
+ jsonByteLength: number
13
+ }
14
+
15
+ function isObjectRow(value: unknown): value is Record<string, unknown> {
16
+ return typeof value === 'object' && value !== null && !Array.isArray(value)
17
+ }
18
+
19
+ function rowsPayloadByteLength(rowCount: number, rowsByteLength: number): number {
20
+ return Buffer.byteLength('{"rows":[]}', 'utf8') +
21
+ rowsByteLength +
22
+ Math.max(0, rowCount - 1)
23
+ }
24
+
25
+ function formatMb(byteLength: number): string {
26
+ return (byteLength / (1024 * 1024)).toFixed(1)
27
+ }
28
+
29
+ function mapReadError(filePath: string, error: unknown): Error {
30
+ const maybeError = error as NodeJS.ErrnoException
31
+
32
+ if (!maybeError.code && error instanceof Error) {
33
+ return error
34
+ }
35
+
36
+ if (maybeError.code === 'ENOENT') {
37
+ return new Error(
38
+ `File not found: ${filePath}. Check the path and filename, then retry.`
39
+ )
40
+ }
41
+
42
+ if (maybeError.code === 'EPERM' || maybeError.code === 'EACCES') {
43
+ return new Error(
44
+ `Cannot read file: ${filePath}. macOS may be blocking access to this folder (for example Downloads). Grant folder permission to your terminal app and retry.`
45
+ )
46
+ }
47
+
48
+ return new Error(`Failed to read file ${filePath}: ${maybeError.message}`)
49
+ }
50
+
51
+ export function estimateRowsPayloadByteLength(rows: Array<Record<string, unknown>>): number {
52
+ const rowsByteLength = rows.reduce((total, row) => {
53
+ return total + Buffer.byteLength(JSON.stringify(row), 'utf8')
54
+ }, 0)
55
+
56
+ return rowsPayloadByteLength(rows.length, rowsByteLength)
57
+ }
58
+
59
+ export async function* streamJsonlRowChunks(
60
+ filePath: string,
61
+ options: JsonlChunkOptions = {}
62
+ ): AsyncGenerator<Array<Record<string, unknown>>> {
63
+ const maxPayloadBytes = options.maxPayloadBytes ?? JSONL_UPLOAD_CHUNK_MAX_BYTES
64
+ let chunk: PendingRow[] = []
65
+ let chunkRowsByteLength = 0
66
+ let parsedRowCount = 0
67
+ let lineNumber = 0
68
+
69
+ const stream = createReadStream(filePath, { encoding: 'utf8' })
70
+ const lines = createInterface({
71
+ input: stream,
72
+ crlfDelay: Infinity,
73
+ })
74
+
75
+ try {
76
+ for await (const line of lines) {
77
+ lineNumber += 1
78
+ const trimmed = line.trim()
79
+ if (!trimmed) {
80
+ continue
81
+ }
82
+
83
+ let parsed: unknown
84
+ try {
85
+ parsed = JSON.parse(trimmed)
86
+ } catch {
87
+ throw new Error(`Invalid JSONL at line ${lineNumber}`)
88
+ }
89
+
90
+ if (!isObjectRow(parsed)) {
91
+ throw new Error(`Row ${lineNumber} must be a JSON object`)
92
+ }
93
+
94
+ const jsonByteLength = Buffer.byteLength(JSON.stringify(parsed), 'utf8')
95
+ const singleRowPayloadBytes = rowsPayloadByteLength(1, jsonByteLength)
96
+ if (singleRowPayloadBytes > maxPayloadBytes) {
97
+ throw new Error(
98
+ `Row ${lineNumber} is ${formatMb(singleRowPayloadBytes)} MB after JSON encoding, which exceeds the ${formatMb(maxPayloadBytes)} MB upload chunk limit`
99
+ )
100
+ }
101
+
102
+ const nextPayloadBytes = rowsPayloadByteLength(
103
+ chunk.length + 1,
104
+ chunkRowsByteLength + jsonByteLength
105
+ )
106
+
107
+ if (chunk.length > 0 && nextPayloadBytes > maxPayloadBytes) {
108
+ yield chunk.map(item => item.row)
109
+ chunk = []
110
+ chunkRowsByteLength = 0
111
+ }
112
+
113
+ chunk.push({ row: parsed, jsonByteLength })
114
+ chunkRowsByteLength += jsonByteLength
115
+ parsedRowCount += 1
116
+ }
117
+ } catch (error) {
118
+ throw mapReadError(filePath, error)
119
+ } finally {
120
+ lines.close()
121
+ stream.destroy()
122
+ }
123
+
124
+ if (chunk.length > 0) {
125
+ yield chunk.map(item => item.row)
126
+ }
127
+
128
+ if (parsedRowCount === 0) {
129
+ throw new Error('Dataset file contains no rows')
130
+ }
131
+ }