orizu 0.5.39 → 0.5.40
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +77 -0
- package/dist/dataset-commands.d.ts +3 -1
- package/dist/dataset-commands.d.ts.map +1 -1
- package/dist/dataset-commands.js +101 -62
- package/dist/dataset-delete-workflow.d.ts +4 -0
- package/dist/dataset-delete-workflow.d.ts.map +1 -0
- package/dist/dataset-delete-workflow.js +75 -0
- package/dist/dataset-lock-workflow.d.ts +12 -0
- package/dist/dataset-lock-workflow.d.ts.map +1 -0
- package/dist/dataset-lock-workflow.js +76 -0
- package/dist/dataset-readme-workflow.d.ts +13 -0
- package/dist/dataset-readme-workflow.d.ts.map +1 -0
- package/dist/dataset-readme-workflow.js +82 -0
- package/dist/dataset-row-workflow.d.ts +14 -0
- package/dist/dataset-row-workflow.d.ts.map +1 -0
- package/dist/dataset-row-workflow.js +202 -0
- package/dist/dataset-split-reads.d.ts +28 -0
- package/dist/dataset-split-reads.d.ts.map +1 -0
- package/dist/dataset-split-reads.js +47 -0
- package/dist/help.d.ts.map +1 -1
- package/dist/help.js +29 -3
- package/dist/hosted-boot.d.ts +1 -0
- package/dist/hosted-boot.d.ts.map +1 -1
- package/dist/hosted-boot.js +17 -0
- package/dist/hosted-harness-opencode.d.ts +4 -5
- package/dist/hosted-harness-opencode.d.ts.map +1 -1
- package/dist/hosted-harness-opencode.js +66 -52
- package/dist/hosted-loop-lifecycle.d.ts +4 -10
- package/dist/hosted-loop-lifecycle.d.ts.map +1 -1
- package/dist/hosted-loop-lifecycle.js +4 -10
- package/dist/hosted-loop.d.ts +2 -0
- package/dist/hosted-loop.d.ts.map +1 -1
- package/dist/hosted-loop.js +5 -4
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +9 -1
- package/dist/instruction-delivery.d.ts +14 -0
- package/dist/instruction-delivery.d.ts.map +1 -0
- package/dist/instruction-delivery.js +103 -0
- package/dist/instruction-set-sync/index.d.ts +17 -0
- package/dist/instruction-set-sync/index.d.ts.map +1 -1
- package/dist/instruction-set-sync/index.js +44 -17
- package/dist/instruction-set-update/index.d.ts +2 -2
- package/dist/instruction-set-update/index.d.ts.map +1 -1
- package/dist/instruction-set-update/index.js +2 -2
- package/dist/instruction-sets-cli.d.ts.map +1 -1
- package/dist/instruction-sets-cli.js +19 -27
- package/dist/json-response.d.ts +1 -0
- package/dist/json-response.d.ts.map +1 -1
- package/dist/json-response.js +7 -0
- package/dist/preview-runtime.d.ts +2 -2
- package/dist/preview-runtime.d.ts.map +1 -1
- package/dist/preview-runtime.js +37 -7
- package/dist/task-export-workflow.d.ts +4 -0
- package/dist/task-export-workflow.d.ts.map +1 -0
- package/dist/task-export-workflow.js +46 -0
- package/dist/workbench-cli.js +1 -1
- package/package.json +1 -1
- package/src/dataset-commands.ts +92 -92
- package/src/dataset-delete-workflow.ts +72 -0
- package/src/dataset-lock-workflow.ts +74 -0
- package/src/dataset-readme-workflow.ts +80 -0
- package/src/dataset-row-workflow.ts +248 -0
- package/src/dataset-split-reads.ts +43 -0
- package/src/help.ts +29 -3
- package/src/hosted-boot.ts +16 -1
- package/src/hosted-harness-opencode.ts +67 -65
- package/src/hosted-loop-lifecycle.ts +4 -10
- package/src/hosted-loop.ts +6 -4
- package/src/hosted-model-cache.d.mts +2 -0
- package/src/hosted-model-cache.mjs +53 -0
- package/src/index.ts +10 -0
- package/src/instruction-delivery.ts +123 -0
- package/src/instruction-set-sync/index.ts +61 -14
- package/src/instruction-set-update/index.ts +4 -3
- package/src/instruction-sets-cli.ts +19 -27
- package/src/json-response.ts +8 -0
- package/src/preview-runtime.ts +40 -10
- package/src/task-export-workflow.ts +56 -0
- package/src/workbench-cli.ts +1 -1
- package/vendor/skills/orizu/references/cli-reference.md +34 -0
- package/vendor/skills/orizu/references/dataset-design.md +1 -1
- package/dist/dataset-edit-rows.d.ts +0 -10
- package/dist/dataset-edit-rows.d.ts.map +0 -1
- package/dist/dataset-edit-rows.js +0 -90
- package/src/dataset-edit-rows.ts +0 -124
package/README.md
CHANGED
|
@@ -336,6 +336,10 @@ orizu logout
|
|
|
336
336
|
<td>Download a dataset</td>
|
|
337
337
|
<td><code>orizu datasets download --dataset <datasetId> --format jsonl --out ./dataset.jsonl</code></td>
|
|
338
338
|
</tr>
|
|
339
|
+
<tr>
|
|
340
|
+
<td>Inspect saved splits and download validation rows</td>
|
|
341
|
+
<td><code>orizu datasets splits list <datasetVersionId> --json</code><br><code>orizu datasets splits show <splitSetId> --json</code><br><code>orizu datasets download --dataset-version <datasetVersionId> --split-set <splitSetId> --split validation --format jsonl --out ./validation.jsonl</code></td>
|
|
342
|
+
</tr>
|
|
339
343
|
<tr>
|
|
340
344
|
<td>Append dataset rows</td>
|
|
341
345
|
<td><code>orizu datasets append --dataset <datasetId> --file ./datasets/support-extra.jsonl</code></td>
|
|
@@ -497,6 +501,39 @@ or interactive selection. SIGINT/SIGTERM closes append's owned JSONL reader and
|
|
|
497
501
|
in-flight transport; retained interactive selection keeps ordinary process signal
|
|
498
502
|
termination. Append adds no transient write retries or write deduplication.
|
|
499
503
|
|
|
504
|
+
Dataset `edit-rows` and `delete-rows` use the same transfer deadline for each
|
|
505
|
+
request, including its response body. When a request times out, loses its
|
|
506
|
+
connection or gets a server error or a malformed reply, the message says the
|
|
507
|
+
change may already be applied (or removed): check those rows before retrying.
|
|
508
|
+
A refusal from the server (for example, a locked dataset) leaves `edit-rows`
|
|
509
|
+
with the usual retry advice, and reports `delete-rows` as refused.
|
|
510
|
+
SIGINT/SIGTERM aborts the request in flight and exits 130/143. Neither command
|
|
511
|
+
retries a write automatically.
|
|
512
|
+
|
|
513
|
+
Dataset `lock` uses the same transfer deadline for its request, including the
|
|
514
|
+
response body. After a timeout, a lost connection, a server error or a malformed
|
|
515
|
+
reply, the message says the dataset may already be locked: running `lock` again
|
|
516
|
+
is safe and shows its current state. A refusal from the server keeps its usual
|
|
517
|
+
message. Lock keeps ordinary process signal termination and never retries.
|
|
518
|
+
|
|
519
|
+
Dataset `delete` asks you to type the dataset id first, then uses the same
|
|
520
|
+
transfer deadline for its request, including the response body. After a
|
|
521
|
+
timeout, a lost connection, a server error or a malformed reply, the message
|
|
522
|
+
says the dataset may already be deleted: check with
|
|
523
|
+
`orizu datasets list --status all` before retrying. When tasks or app versions
|
|
524
|
+
still use the dataset, the refusal also names `orizu datasets archive <id>`.
|
|
525
|
+
Delete keeps ordinary process signal termination and never retries.
|
|
526
|
+
|
|
527
|
+
Dataset `readme set` uses the same transfer deadline for its request, including
|
|
528
|
+
the response body. Each save adds a new README version. After a timeout, a lost
|
|
529
|
+
connection, a server error or a malformed reply, the message says the README may
|
|
530
|
+
already be saved as a new version: check the dataset's README tab before saving
|
|
531
|
+
again, since another save would add a second version. When another save for the
|
|
532
|
+
same dataset happened at the same time, the message says yours was not saved
|
|
533
|
+
and to run the command again. A README file that cannot be read is named and
|
|
534
|
+
nothing is sent. `readme set` keeps ordinary process signal termination and
|
|
535
|
+
never retries.
|
|
536
|
+
|
|
500
537
|
|
|
501
538
|
## Interactive And Automated Usage
|
|
502
539
|
|
|
@@ -514,3 +551,43 @@ Use `--json` on supported task and app commands when automation needs structured
|
|
|
514
551
|
- [skills/orizu/references/cli-reference.md](skills/orizu/references/cli-reference.md): compact command matrix and end-to-end flows
|
|
515
552
|
|
|
516
553
|
If the server loses or cannot decode a database append acknowledgement, it preserves the request's prepared row artifacts because rows may already reference them. Unknown outcomes require inspection; automatic orphan reconciliation and write deduplication are not promised. Known SQL refusals clean up only that request's newly prepared artifacts.
|
|
554
|
+
|
|
555
|
+
### Enrich validation rows while preserving split assignments
|
|
556
|
+
|
|
557
|
+
Start with the immutable dataset version ID returned by dataset upload or version
|
|
558
|
+
creation. List its saved split sets, then inspect one set to find the exact
|
|
559
|
+
partition name (custom names and empty partitions are retained):
|
|
560
|
+
|
|
561
|
+
```bash
|
|
562
|
+
orizu datasets splits list DATASET_VERSION_ID --json
|
|
563
|
+
orizu datasets splits show SPLIT_SET_ID --json > split.json
|
|
564
|
+
orizu datasets download --dataset-version DATASET_VERSION_ID --split-set SPLIT_SET_ID --split validation --format jsonl --out validation.jsonl
|
|
565
|
+
```
|
|
566
|
+
|
|
567
|
+
`validation.jsonl` contains only that partition, in its saved order. Each row has
|
|
568
|
+
its canonical `id` and the fields stored in that version. These reads stay fixed
|
|
569
|
+
when the live dataset changes. No prompt, runner, scorer or optimization is needed.
|
|
570
|
+
|
|
571
|
+
The assignment JSON retains set and partition metadata, including arrays, scalars
|
|
572
|
+
and null values, together with exact saved memberships.
|
|
573
|
+
|
|
574
|
+
Enrich the exported rows in your own workflow, keeping every `id` unchanged.
|
|
575
|
+
Combine them with the other rows when preparing the complete enriched dataset.
|
|
576
|
+
After uploading that dataset or creating its version, use the new version's ID
|
|
577
|
+
explicitly to reuse every original assignment:
|
|
578
|
+
|
|
579
|
+
```bash
|
|
580
|
+
orizu datasets splits create TARGET_DATASET_VERSION_ID --from-file split.json --json
|
|
581
|
+
```
|
|
582
|
+
|
|
583
|
+
The import accepts the `name`, `strategy`, `seed`, `metadata` and
|
|
584
|
+
`partitions: [{name, row_ids}]` from show JSON. Source identity is for inspection;
|
|
585
|
+
the command's target version decides where assignments are created. Every
|
|
586
|
+
referenced canonical row ID must exist in that target. Missing IDs fail instead
|
|
587
|
+
of being dropped or remapped. Unknown or inaccessible versions, sets or names,
|
|
588
|
+
mismatched selectors and incomplete artifacts fail; pinned reads never fall back
|
|
589
|
+
to live data. Existing live downloads remain available with `--dataset`.
|
|
590
|
+
|
|
591
|
+
For saved split names beginning with `--`, use the unambiguous equals spelling:
|
|
592
|
+
`orizu datasets download --dataset-version DATASET_VERSION_ID --split-set SPLIT_SET_ID --split=--validation --format jsonl --out validation.jsonl`.
|
|
593
|
+
The same spelling supports `--split=--json` and `--split=--help`; all pinned selectors remain required.
|
|
@@ -1,4 +1,6 @@
|
|
|
1
1
|
declare function createDatasetVersion(): Promise<void>;
|
|
2
|
+
declare function listDatasetSplitSets(): Promise<void>;
|
|
3
|
+
declare function showDatasetSplitSet(): Promise<void>;
|
|
2
4
|
declare function createDatasetSplitSet(): Promise<void>;
|
|
3
5
|
declare function listDatasets(): Promise<void>;
|
|
4
6
|
declare function uploadDataset(): Promise<void>;
|
|
@@ -12,5 +14,5 @@ declare function deleteDataset(): Promise<void>;
|
|
|
12
14
|
declare function lockDataset(): Promise<void>;
|
|
13
15
|
declare function cloneDataset(): Promise<void>;
|
|
14
16
|
declare function downloadAnnotations(): Promise<void>;
|
|
15
|
-
export { uploadDataset, listDatasets, pushDataset, downloadDataset, appendDatasetRows, setDatasetReadme, createDatasetVersion, createDatasetSplitSet, editDatasetRows, deleteDatasetRows, deleteDataset, lockDataset, cloneDataset, downloadAnnotations, };
|
|
17
|
+
export { uploadDataset, listDatasets, pushDataset, downloadDataset, appendDatasetRows, setDatasetReadme, createDatasetVersion, createDatasetSplitSet, listDatasetSplitSets, showDatasetSplitSet, editDatasetRows, deleteDatasetRows, deleteDataset, lockDataset, cloneDataset, downloadAnnotations, };
|
|
16
18
|
//# sourceMappingURL=dataset-commands.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"dataset-commands.d.ts","sourceRoot":"","sources":["../src/dataset-commands.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"dataset-commands.d.ts","sourceRoot":"","sources":["../src/dataset-commands.ts"],"names":[],"mappings":"AAuJA,iBAAe,oBAAoB,kBAiClC;AAED,iBAAe,oBAAoB,kBAYlC;AAED,iBAAe,mBAAmB,kBAcjC;AAED,iBAAe,qBAAqB,kBAkDnC;AAED,iBAAe,YAAY,kBAW1B;AAsBD,iBAAe,aAAa,kBAiC3B;AAED,iBAAe,WAAW,kBAwCzB;AAED,iBAAe,gBAAgB,kBA+B9B;AAgBD,iBAAe,eAAe,kBA6C7B;AAED,iBAAe,iBAAiB,kBAU/B;AAED,iBAAe,eAAe,kBA2B7B;AAED,iBAAe,iBAAiB,kBA2B/B;AA0BD,iBAAe,aAAa,kBAuB3B;AAED,iBAAe,WAAW,kBAuBzB;AAED,iBAAe,YAAY,kBAuC1B;AAED,iBAAe,mBAAmB,kBA0BjC;AAED,OAAO,EACL,aAAa,EACb,YAAY,EACZ,WAAW,EACX,eAAe,EACf,iBAAiB,EACjB,gBAAgB,EAChB,oBAAoB,EACpB,qBAAqB,EACrB,oBAAoB,EACpB,mBAAmB,EACnB,eAAe,EACf,iBAAiB,EACjB,aAAa,EACb,WAAW,EACX,YAAY,EACZ,mBAAmB,GACpB,CAAA"}
|
package/dist/dataset-commands.js
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
// ORI-2044: moved from index.ts to keep dataset commands together.
|
|
2
|
-
import { readFileSync
|
|
2
|
+
import { readFileSync } from 'fs';
|
|
3
3
|
import { basename, extname } from 'path';
|
|
4
4
|
import { createInterface } from 'readline/promises';
|
|
5
5
|
import { stdin as input, stdout as output } from 'process';
|
|
6
|
-
import { getArg, expandHomePath, getPositionalArg, hasJsonFlag, getArchiveListStatus, cliArgs, isInteractiveTerminal, } from './command-line-arguments.js';
|
|
6
|
+
import { getArg, hasArg, expandHomePath, getPositionalArg, hasJsonFlag, getArchiveListStatus, cliArgs, isInteractiveTerminal, } from './command-line-arguments.js';
|
|
7
7
|
import { resolveProjectSlug, fetchDatasets, selectDatasetInteractively, selectTaskIdInteractively, } from './account-directory.js';
|
|
8
8
|
import { sanitizeTerminalText, parseJsonResponse } from './json-response.js';
|
|
9
9
|
import { authedFetch } from './http.js';
|
|
@@ -14,9 +14,18 @@ import { printDatasetSummaries } from './archive-artifacts-cli.js';
|
|
|
14
14
|
import { formatTerminalLink } from './auth-commands.js';
|
|
15
15
|
import { parseDatasetFile } from './file-parser.js';
|
|
16
16
|
import { parseDatasetReference } from './dataset-download.js';
|
|
17
|
-
import { editDatasetRows as runEditDatasetRows } from './dataset-edit-rows.js';
|
|
18
17
|
import { parseCommaSeparated } from './task-commands.js';
|
|
19
18
|
const MAX_README_LENGTH = 200_000;
|
|
19
|
+
function readReadmeFile(readmeFile) {
|
|
20
|
+
try {
|
|
21
|
+
return readFileSync(expandHomePath(readmeFile), 'utf8');
|
|
22
|
+
}
|
|
23
|
+
catch (error) {
|
|
24
|
+
const missing = error instanceof Error && 'code' in error && error.code === 'ENOENT';
|
|
25
|
+
const reason = missing ? 'file not found' : error instanceof Error ? error.message : String(error);
|
|
26
|
+
throw new Error(`Could not read README file ${readmeFile}: ${reason}`);
|
|
27
|
+
}
|
|
28
|
+
}
|
|
20
29
|
function readReadmeMarkdownFromArgs(usage) {
|
|
21
30
|
const readmeFile = getArg('--readme-file');
|
|
22
31
|
const readmeText = getArg('--readme-text');
|
|
@@ -24,7 +33,7 @@ function readReadmeMarkdownFromArgs(usage) {
|
|
|
24
33
|
throw new Error(`Use either --readme-file or --readme-text, not both.\n${usage}`);
|
|
25
34
|
}
|
|
26
35
|
const markdown = readmeFile !== null
|
|
27
|
-
?
|
|
36
|
+
? readReadmeFile(readmeFile)
|
|
28
37
|
: readmeText;
|
|
29
38
|
if (markdown !== null && markdown.length > MAX_README_LENGTH) {
|
|
30
39
|
throw new Error(`README markdown is too large. Maximum size is ${MAX_README_LENGTH} characters.`);
|
|
@@ -116,6 +125,41 @@ async function createDatasetVersion() {
|
|
|
116
125
|
printLine(`Created dataset version ${sanitizeTerminalText(data.datasetVersion.id)}` +
|
|
117
126
|
(details ? ` (${sanitizeTerminalText(details)})` : ''));
|
|
118
127
|
}
|
|
128
|
+
async function listDatasetSplitSets() {
|
|
129
|
+
const versionId = getPositionalArg(3);
|
|
130
|
+
if (!versionId)
|
|
131
|
+
throw new Error('Usage: orizu datasets splits list <datasetVersionId> [--json]');
|
|
132
|
+
const { listDatasetSplits } = await import('./dataset-split-reads.js');
|
|
133
|
+
const data = await listDatasetSplits(versionId);
|
|
134
|
+
if (hasJsonFlag())
|
|
135
|
+
return printJson({ ...data });
|
|
136
|
+
printLine(`Dataset version ${sanitizeTerminalText(data.dataset_version_id)}`);
|
|
137
|
+
for (const set of data.split_sets) {
|
|
138
|
+
printLine(`${sanitizeTerminalText(set.name)} (${sanitizeTerminalText(set.id)})`);
|
|
139
|
+
for (const partition of set.partitions)
|
|
140
|
+
printLine(` ${sanitizeTerminalText(partition.name)}: ${partition.row_count} rows`);
|
|
141
|
+
}
|
|
142
|
+
if (!data.split_sets.length)
|
|
143
|
+
printLine('No saved split sets.');
|
|
144
|
+
}
|
|
145
|
+
async function showDatasetSplitSet() {
|
|
146
|
+
const setId = getPositionalArg(3);
|
|
147
|
+
if (!setId)
|
|
148
|
+
throw new Error('Usage: orizu datasets splits show <splitSetId> [--json]');
|
|
149
|
+
const { showDatasetSplit } = await import('./dataset-split-reads.js');
|
|
150
|
+
const data = await showDatasetSplit(setId);
|
|
151
|
+
if (hasJsonFlag())
|
|
152
|
+
return printJson({ ...data });
|
|
153
|
+
printLine(`${sanitizeTerminalText(data.name)} (${sanitizeTerminalText(data.split_set_id)})`);
|
|
154
|
+
printLine(`Dataset version: ${sanitizeTerminalText(data.dataset_version_id)}`);
|
|
155
|
+
printLine(`Strategy: ${sanitizeTerminalText(data.strategy)}; seed: ${data.seed ?? 'none'}`);
|
|
156
|
+
printLine(`Metadata: ${sanitizeTerminalText(JSON.stringify(data.metadata))}`);
|
|
157
|
+
for (const partition of data.partitions) {
|
|
158
|
+
printLine(`${sanitizeTerminalText(partition.name)}: ${partition.row_ids.length} rows`);
|
|
159
|
+
for (const id of partition.row_ids)
|
|
160
|
+
printLine(` ${sanitizeTerminalText(id)}`);
|
|
161
|
+
}
|
|
162
|
+
}
|
|
119
163
|
async function createDatasetSplitSet() {
|
|
120
164
|
const datasetVersionId = getPositionalArg(3) || getArg('--dataset-version');
|
|
121
165
|
const splitFile = getArg('--from-file');
|
|
@@ -132,7 +176,7 @@ async function createDatasetSplitSet() {
|
|
|
132
176
|
throw new Error('Usage: orizu datasets splits create <datasetVersionId> [--from-file <split.json>] [--name <name>] [--seed <n>] [--train <ratio>] [--validation <ratio>] [--test <ratio>] [--json]');
|
|
133
177
|
}
|
|
134
178
|
const partitions = splitSpec && Array.isArray(splitSpec.partitions) ? splitSpec.partitions : undefined;
|
|
135
|
-
const metadata = splitSpec &&
|
|
179
|
+
const metadata = splitSpec && Object.prototype.hasOwnProperty.call(splitSpec, 'metadata')
|
|
136
180
|
? splitSpec.metadata
|
|
137
181
|
: undefined;
|
|
138
182
|
const response = await authedFetch(`/api/cli/dataset-versions/${encodeURIComponent(datasetVersionId)}/split-sets`, {
|
|
@@ -264,25 +308,18 @@ async function setDatasetReadme() {
|
|
|
264
308
|
throw new Error(usage);
|
|
265
309
|
}
|
|
266
310
|
const resolvedDatasetId = await resolveDatasetIdForProject(datasetId, project);
|
|
267
|
-
const
|
|
268
|
-
|
|
269
|
-
headers: { 'Content-Type': 'application/json' },
|
|
270
|
-
body: JSON.stringify({ markdown: readmeMarkdown }),
|
|
271
|
-
});
|
|
272
|
-
if (!response.ok) {
|
|
273
|
-
throw new Error(`Failed to save dataset README: ${await response.text()}`);
|
|
274
|
-
}
|
|
275
|
-
const data = await parseJsonResponse(response, 'Dataset README save');
|
|
311
|
+
const { saveDatasetReadme } = await import('./dataset-readme-workflow.js');
|
|
312
|
+
const version = await saveDatasetReadme(resolvedDatasetId, readmeMarkdown);
|
|
276
313
|
if (hasJsonFlag()) {
|
|
277
314
|
printJson({
|
|
278
|
-
dataset_id:
|
|
279
|
-
readme_version_id:
|
|
280
|
-
readme_version_num:
|
|
315
|
+
dataset_id: version.dataset_id,
|
|
316
|
+
readme_version_id: version.id,
|
|
317
|
+
readme_version_num: version.version_num,
|
|
281
318
|
});
|
|
282
319
|
return;
|
|
283
320
|
}
|
|
284
|
-
printLine(`Saved README v${
|
|
285
|
-
`${sanitizeTerminalText(
|
|
321
|
+
printLine(`Saved README v${version.version_num} for dataset ` +
|
|
322
|
+
`${sanitizeTerminalText(version.dataset_id)} (${sanitizeTerminalText(version.id)}).`);
|
|
286
323
|
}
|
|
287
324
|
function getDatasetReferenceInput() {
|
|
288
325
|
const fromFlag = getArg('--dataset');
|
|
@@ -303,6 +340,24 @@ async function downloadDataset() {
|
|
|
303
340
|
if (!['csv', 'json', 'jsonl'].includes(format)) {
|
|
304
341
|
throw new Error('format must be one of: csv, json, jsonl');
|
|
305
342
|
}
|
|
343
|
+
const versionId = getArg('--dataset-version'), setId = getArg('--split-set');
|
|
344
|
+
const inlineSplit = cliArgs.find(argument => argument.startsWith('--split='));
|
|
345
|
+
const split = inlineSplit === undefined ? getArg('--split') : inlineSplit.slice('--split='.length);
|
|
346
|
+
if (['--dataset-version', '--split-set', '--split'].some(flag => hasArg(flag) || cliArgs.some(argument => argument.startsWith(`${flag}=`)))) {
|
|
347
|
+
if (!versionId || !setId || !split || [versionId, setId].some(value => value.startsWith('--')) || (inlineSplit === undefined && split.startsWith('--')))
|
|
348
|
+
throw new Error('Pinned download requires --dataset-version, --split-set and --split together');
|
|
349
|
+
if (datasetInput)
|
|
350
|
+
throw new Error('Use --dataset-version with split selectors without a live --dataset or dataset argument');
|
|
351
|
+
if (format === 'csv')
|
|
352
|
+
throw new Error('Pinned download format must be json or jsonl');
|
|
353
|
+
const filename = outPathArg ? expandHomePath(outPathArg) : `${versionId}.${format}`;
|
|
354
|
+
const { downloadPinnedDatasetSplit } = await import('./dataset-split-reads.js');
|
|
355
|
+
await downloadPinnedDatasetSplit(versionId, setId, split, format, filename);
|
|
356
|
+
if (hasJsonFlag())
|
|
357
|
+
return printJson({ dataset_version_id: versionId, split_set_id: setId, split, format, savedTo: filename });
|
|
358
|
+
printLine(`Saved split ${sanitizeTerminalText(split)} from dataset version ${sanitizeTerminalText(versionId)} (${format.toUpperCase()}) to ${sanitizeTerminalText(filename)}`);
|
|
359
|
+
return;
|
|
360
|
+
}
|
|
306
361
|
let datasetId;
|
|
307
362
|
if (datasetInput) {
|
|
308
363
|
datasetId = parseDatasetReference(datasetInput).datasetId;
|
|
@@ -349,7 +404,8 @@ async function editDatasetRows() {
|
|
|
349
404
|
const selected = await selectDatasetInteractively(projectArg);
|
|
350
405
|
datasetId = selected.datasetId;
|
|
351
406
|
}
|
|
352
|
-
await
|
|
407
|
+
const { editDatasetRows: runEditDatasetRows } = await import('./dataset-row-workflow.js');
|
|
408
|
+
const exitCode = await runEditDatasetRows({
|
|
353
409
|
datasetId,
|
|
354
410
|
file: expandHomePath(fileArg),
|
|
355
411
|
json: hasJsonFlag(),
|
|
@@ -357,6 +413,8 @@ async function editDatasetRows() {
|
|
|
357
413
|
printLine,
|
|
358
414
|
sanitize: sanitizeTerminalText,
|
|
359
415
|
});
|
|
416
|
+
if (exitCode !== 0)
|
|
417
|
+
process.exitCode = exitCode;
|
|
360
418
|
}
|
|
361
419
|
async function deleteDatasetRows() {
|
|
362
420
|
const projectArg = getArg('--project');
|
|
@@ -373,22 +431,17 @@ async function deleteDatasetRows() {
|
|
|
373
431
|
const selected = await selectDatasetInteractively(projectArg);
|
|
374
432
|
datasetId = selected.datasetId;
|
|
375
433
|
}
|
|
376
|
-
const
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
434
|
+
const { deleteDatasetRows: runDeleteDatasetRows } = await import('./dataset-row-workflow.js');
|
|
435
|
+
const exitCode = await runDeleteDatasetRows({
|
|
436
|
+
datasetId,
|
|
437
|
+
rowIds,
|
|
438
|
+
json: hasJsonFlag(),
|
|
439
|
+
printJson,
|
|
440
|
+
printLine,
|
|
441
|
+
sanitize: sanitizeTerminalText,
|
|
382
442
|
});
|
|
383
|
-
if (
|
|
384
|
-
|
|
385
|
-
}
|
|
386
|
-
const data = await parseJsonResponse(response, 'Dataset delete rows');
|
|
387
|
-
if (hasJsonFlag()) {
|
|
388
|
-
printJson({ dataset: data.dataset, deletedCount: data.deletedCount });
|
|
389
|
-
return;
|
|
390
|
-
}
|
|
391
|
-
printLine(`Deleted ${data.deletedCount} rows from dataset ${sanitizeTerminalText(data.dataset.name)} (${sanitizeTerminalText(data.dataset.id)}). New row count: ${data.dataset.rowCount}`);
|
|
443
|
+
if (exitCode !== 0)
|
|
444
|
+
process.exitCode = exitCode;
|
|
392
445
|
}
|
|
393
446
|
async function confirmDatasetDeletion(dataset) {
|
|
394
447
|
if (!isInteractiveTerminal()) {
|
|
@@ -419,18 +472,15 @@ async function deleteDataset() {
|
|
|
419
472
|
dataset = await selectDatasetInteractively(projectArg);
|
|
420
473
|
}
|
|
421
474
|
await confirmDatasetDeletion(dataset);
|
|
422
|
-
const
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
throw new Error(`Delete failed: ${await response.text()}`);
|
|
427
|
-
}
|
|
428
|
-
const data = await parseJsonResponse(response, 'Dataset delete');
|
|
475
|
+
const { deleteDataset: requestDelete } = await import('./dataset-delete-workflow.js');
|
|
476
|
+
// Only a dataset picked from a project's list has a known project; with
|
|
477
|
+
// --dataset, --project is never checked against the dataset.
|
|
478
|
+
const deleted = await requestDelete(dataset.datasetId, dataset.project ?? null);
|
|
429
479
|
if (hasJsonFlag()) {
|
|
430
|
-
printJson({ dataset:
|
|
480
|
+
printJson({ dataset: deleted, deleted: true });
|
|
431
481
|
return;
|
|
432
482
|
}
|
|
433
|
-
printLine(`Deleted dataset ${sanitizeTerminalText(
|
|
483
|
+
printLine(`Deleted dataset ${sanitizeTerminalText(deleted.id)}.`);
|
|
434
484
|
}
|
|
435
485
|
async function lockDataset() {
|
|
436
486
|
const projectArg = getArg('--project');
|
|
@@ -444,20 +494,13 @@ async function lockDataset() {
|
|
|
444
494
|
const selected = await selectDatasetInteractively(projectArg);
|
|
445
495
|
datasetId = selected.datasetId;
|
|
446
496
|
}
|
|
447
|
-
const
|
|
448
|
-
|
|
449
|
-
headers: { 'Content-Type': 'application/json' },
|
|
450
|
-
body: JSON.stringify(reason ? { reason } : {}),
|
|
451
|
-
});
|
|
452
|
-
if (!response.ok) {
|
|
453
|
-
throw new Error(`Lock failed: ${await response.text()}`);
|
|
454
|
-
}
|
|
455
|
-
const data = await parseJsonResponse(response, 'Dataset lock');
|
|
497
|
+
const { lockDataset: requestLock } = await import('./dataset-lock-workflow.js');
|
|
498
|
+
const dataset = await requestLock(datasetId, reason);
|
|
456
499
|
if (hasJsonFlag()) {
|
|
457
|
-
printJson({ dataset
|
|
500
|
+
printJson({ dataset });
|
|
458
501
|
return;
|
|
459
502
|
}
|
|
460
|
-
printLine(`Locked dataset ${sanitizeTerminalText(
|
|
503
|
+
printLine(`Locked dataset ${sanitizeTerminalText(dataset.name)} (${sanitizeTerminalText(dataset.id)}) at ${sanitizeTerminalText(dataset.lockedAt)}. Row count: ${dataset.rowCount}`);
|
|
461
504
|
}
|
|
462
505
|
async function cloneDataset() {
|
|
463
506
|
const projectArg = getArg('--project');
|
|
@@ -496,20 +539,16 @@ async function downloadAnnotations() {
|
|
|
496
539
|
if (!taskId) {
|
|
497
540
|
taskId = await selectTaskIdInteractively();
|
|
498
541
|
}
|
|
499
|
-
const response = await authedFetch(`/api/cli/tasks/${taskId}/export?format=${format}`);
|
|
500
|
-
if (!response.ok) {
|
|
501
|
-
throw new Error(`Download failed: ${await response.text()}`);
|
|
502
|
-
}
|
|
503
542
|
const fallbackName = `${taskId}.${format}`;
|
|
504
543
|
const filename = outPathArg
|
|
505
544
|
? expandHomePath(outPathArg)
|
|
506
545
|
: fallbackName;
|
|
507
|
-
const
|
|
508
|
-
|
|
546
|
+
const { runTaskExportDownload } = await import('./task-export-workflow.js');
|
|
547
|
+
await runTaskExportDownload(taskId, format, filename);
|
|
509
548
|
if (hasJsonFlag()) {
|
|
510
549
|
printJson({ taskId, format, savedTo: filename });
|
|
511
550
|
return;
|
|
512
551
|
}
|
|
513
552
|
printLine(`Saved ${format.toUpperCase()} export to ${sanitizeTerminalText(filename)}`);
|
|
514
553
|
}
|
|
515
|
-
export { uploadDataset, listDatasets, pushDataset, downloadDataset, appendDatasetRows, setDatasetReadme, createDatasetVersion, createDatasetSplitSet, editDatasetRows, deleteDatasetRows, deleteDataset, lockDataset, cloneDataset, downloadAnnotations, };
|
|
554
|
+
export { uploadDataset, listDatasets, pushDataset, downloadDataset, appendDatasetRows, setDatasetReadme, createDatasetVersion, createDatasetSplitSet, listDatasetSplitSets, showDatasetSplitSet, editDatasetRows, deleteDatasetRows, deleteDataset, lockDataset, cloneDataset, downloadAnnotations, };
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"dataset-delete-workflow.d.ts","sourceRoot":"","sources":["../src/dataset-delete-workflow.ts"],"names":[],"mappings":"AAmEA,wBAAsB,aAAa,CAAC,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,IAAI,GAAG,OAAO,CAAC;IAAE,EAAE,EAAE,MAAM,CAAA;CAAE,CAAC,CAItG"}
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
import { Cause, Effect, Exit, Result, Schema } from 'effect';
|
|
2
|
+
import { datasetTransferTimeoutMs } from './dataset-transfer-policy.js';
|
|
3
|
+
import { authedFetch } from './http.js';
|
|
4
|
+
import { parseJsonResponse } from './json-response.js';
|
|
5
|
+
// Loaded only through `await import(...)` from dataset-commands, like append.
|
|
6
|
+
const MAY_BE_DELETED = 'The dataset may already be deleted: check with `orizu datasets list --status all` before retrying.';
|
|
7
|
+
const DeleteReply = Schema.Struct({ dataset: Schema.Struct({ id: Schema.String }) });
|
|
8
|
+
// A refusal (4xx) means nothing was deleted. After any other failure once the
|
|
9
|
+
// request was sent, the delete may have committed, and it cannot be undone.
|
|
10
|
+
class DeleteFailure extends Error {
|
|
11
|
+
mayBeDeleted;
|
|
12
|
+
constructor(message, mayBeDeleted) {
|
|
13
|
+
super(message);
|
|
14
|
+
this.mayBeDeleted = mayBeDeleted;
|
|
15
|
+
}
|
|
16
|
+
}
|
|
17
|
+
function deleteRequest(datasetId, project, timeoutMs) {
|
|
18
|
+
return Effect.suspend(() => {
|
|
19
|
+
let sent = false;
|
|
20
|
+
let timedOut = false;
|
|
21
|
+
return Effect.tryPromise({
|
|
22
|
+
try: async (signal) => {
|
|
23
|
+
const request = new AbortController();
|
|
24
|
+
const abort = () => request.abort();
|
|
25
|
+
const timer = setTimeout(() => { timedOut = true; request.abort(); }, timeoutMs);
|
|
26
|
+
if (signal.aborted)
|
|
27
|
+
abort();
|
|
28
|
+
else
|
|
29
|
+
signal.addEventListener('abort', abort, { once: true });
|
|
30
|
+
try {
|
|
31
|
+
const response = await authedFetch(`/api/cli/datasets/${encodeURIComponent(datasetId)}`, {
|
|
32
|
+
method: 'DELETE',
|
|
33
|
+
signal: request.signal,
|
|
34
|
+
}, () => { sent = true; });
|
|
35
|
+
if (!response.ok) {
|
|
36
|
+
const message = `Delete failed: ${await response.text()}`;
|
|
37
|
+
// A dependency refusal: archiving keeps the dataset and its references.
|
|
38
|
+
if (response.status === 409) {
|
|
39
|
+
const archive = `orizu datasets archive ${datasetId}${project ? ` --project ${project}` : ''}`;
|
|
40
|
+
throw new DeleteFailure(`${message}\nArchive it instead: \`${archive}\``, false);
|
|
41
|
+
}
|
|
42
|
+
throw new DeleteFailure(message, response.status < 400 || response.status >= 500);
|
|
43
|
+
}
|
|
44
|
+
const data = await parseJsonResponse(response, 'Dataset delete');
|
|
45
|
+
const decoded = Schema.decodeUnknownResult(DeleteReply)(data);
|
|
46
|
+
if (Result.isFailure(decoded)) {
|
|
47
|
+
throw new DeleteFailure(`Dataset delete returned an unexpected response (status ${response.status}).`, true);
|
|
48
|
+
}
|
|
49
|
+
return decoded.success.dataset;
|
|
50
|
+
}
|
|
51
|
+
finally {
|
|
52
|
+
clearTimeout(timer);
|
|
53
|
+
signal.removeEventListener('abort', abort);
|
|
54
|
+
request.abort();
|
|
55
|
+
}
|
|
56
|
+
},
|
|
57
|
+
catch: cause => {
|
|
58
|
+
if (timedOut)
|
|
59
|
+
return new DeleteFailure('DATASET_TRANSFER_TIMEOUT: Dataset delete exceeded its deadline', true);
|
|
60
|
+
if (cause instanceof DeleteFailure)
|
|
61
|
+
return cause;
|
|
62
|
+
return new DeleteFailure(cause instanceof Error ? cause.message : String(cause), sent);
|
|
63
|
+
},
|
|
64
|
+
});
|
|
65
|
+
}).pipe(Effect.mapError(failure => new Error(failure.mayBeDeleted ? `${failure.message}\n${MAY_BE_DELETED}` : failure.message)));
|
|
66
|
+
}
|
|
67
|
+
// Runs after the typed confirmation. Nothing to clean up, so signals keep
|
|
68
|
+
// Node's default handling.
|
|
69
|
+
// `project` (team/project), when known, goes into the archive hint.
|
|
70
|
+
export async function deleteDataset(datasetId, project) {
|
|
71
|
+
const exit = await Effect.runPromiseExit(deleteRequest(datasetId, project, datasetTransferTimeoutMs()));
|
|
72
|
+
if (Exit.isSuccess(exit))
|
|
73
|
+
return exit.value;
|
|
74
|
+
throw Cause.squash(exit.cause);
|
|
75
|
+
}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import { Schema } from 'effect';
|
|
2
|
+
declare const LockedDataset: Schema.Struct<{
|
|
3
|
+
readonly id: Schema.String;
|
|
4
|
+
readonly name: Schema.String;
|
|
5
|
+
readonly rowCount: Schema.Number;
|
|
6
|
+
readonly lockedAt: Schema.String;
|
|
7
|
+
readonly lockedBy: Schema.NullOr<Schema.String>;
|
|
8
|
+
}>;
|
|
9
|
+
export type LockedDataset = typeof LockedDataset.Type;
|
|
10
|
+
export declare function lockDataset(datasetId: string, reason: string | null): Promise<LockedDataset>;
|
|
11
|
+
export {};
|
|
12
|
+
//# sourceMappingURL=dataset-lock-workflow.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"dataset-lock-workflow.d.ts","sourceRoot":"","sources":["../src/dataset-lock-workflow.ts"],"names":[],"mappings":"AAAA,OAAO,EAA+B,MAAM,EAAE,MAAM,QAAQ,CAAA;AAU5D,QAAA,MAAM,aAAa;;;;;;EAMjB,CAAA;AAEF,MAAM,MAAM,aAAa,GAAG,OAAO,aAAa,CAAC,IAAI,CAAA;AAmDrD,wBAAsB,WAAW,CAAC,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,IAAI,GAAG,OAAO,CAAC,aAAa,CAAC,CAIlG"}
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
import { Cause, Effect, Exit, Result, Schema } from 'effect';
|
|
2
|
+
import { datasetTransferTimeoutMs } from './dataset-transfer-policy.js';
|
|
3
|
+
import { authedFetch } from './http.js';
|
|
4
|
+
import { parseJsonResponse } from './json-response.js';
|
|
5
|
+
// Loaded only through `await import(...)` from dataset-commands, like append.
|
|
6
|
+
const MAY_BE_LOCKED = 'The dataset may already be locked: running `orizu datasets lock` again is safe and shows its current state.';
|
|
7
|
+
const LockedDataset = Schema.Struct({
|
|
8
|
+
id: Schema.String,
|
|
9
|
+
name: Schema.String,
|
|
10
|
+
rowCount: Schema.Number,
|
|
11
|
+
lockedAt: Schema.String,
|
|
12
|
+
lockedBy: Schema.NullOr(Schema.String),
|
|
13
|
+
});
|
|
14
|
+
const LockReply = Schema.Struct({ dataset: LockedDataset });
|
|
15
|
+
// A refusal (4xx) means nothing was locked. After any other failure once the
|
|
16
|
+
// request was sent, the lock may have committed; locking again is harmless.
|
|
17
|
+
class LockFailure extends Error {
|
|
18
|
+
mayBeLocked;
|
|
19
|
+
constructor(message, mayBeLocked) {
|
|
20
|
+
super(message);
|
|
21
|
+
this.mayBeLocked = mayBeLocked;
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
function lockRequest(datasetId, reason, timeoutMs) {
|
|
25
|
+
return Effect.suspend(() => {
|
|
26
|
+
let sent = false;
|
|
27
|
+
let timedOut = false;
|
|
28
|
+
return Effect.tryPromise({
|
|
29
|
+
try: async (signal) => {
|
|
30
|
+
const request = new AbortController();
|
|
31
|
+
const abort = () => request.abort();
|
|
32
|
+
const timer = setTimeout(() => { timedOut = true; request.abort(); }, timeoutMs);
|
|
33
|
+
if (signal.aborted)
|
|
34
|
+
abort();
|
|
35
|
+
else
|
|
36
|
+
signal.addEventListener('abort', abort, { once: true });
|
|
37
|
+
try {
|
|
38
|
+
const response = await authedFetch(`/api/cli/datasets/${encodeURIComponent(datasetId)}/lock`, {
|
|
39
|
+
method: 'POST',
|
|
40
|
+
headers: { 'Content-Type': 'application/json' },
|
|
41
|
+
body: JSON.stringify(reason ? { reason } : {}),
|
|
42
|
+
signal: request.signal,
|
|
43
|
+
}, () => { sent = true; });
|
|
44
|
+
if (!response.ok) {
|
|
45
|
+
throw new LockFailure(`Lock failed: ${await response.text()}`, response.status < 400 || response.status >= 500);
|
|
46
|
+
}
|
|
47
|
+
const data = await parseJsonResponse(response, 'Dataset lock');
|
|
48
|
+
const decoded = Schema.decodeUnknownResult(LockReply)(data);
|
|
49
|
+
if (Result.isFailure(decoded)) {
|
|
50
|
+
throw new LockFailure(`Dataset lock returned an unexpected response (status ${response.status}).`, true);
|
|
51
|
+
}
|
|
52
|
+
return decoded.success.dataset;
|
|
53
|
+
}
|
|
54
|
+
finally {
|
|
55
|
+
clearTimeout(timer);
|
|
56
|
+
signal.removeEventListener('abort', abort);
|
|
57
|
+
request.abort();
|
|
58
|
+
}
|
|
59
|
+
},
|
|
60
|
+
catch: cause => {
|
|
61
|
+
if (timedOut)
|
|
62
|
+
return new LockFailure('DATASET_TRANSFER_TIMEOUT: Dataset lock exceeded its deadline', true);
|
|
63
|
+
if (cause instanceof LockFailure)
|
|
64
|
+
return cause;
|
|
65
|
+
return new LockFailure(cause instanceof Error ? cause.message : String(cause), sent);
|
|
66
|
+
},
|
|
67
|
+
});
|
|
68
|
+
}).pipe(Effect.mapError(failure => new Error(failure.mayBeLocked ? `${failure.message}\n${MAY_BE_LOCKED}` : failure.message)));
|
|
69
|
+
}
|
|
70
|
+
// Nothing to clean up, so signals keep Node's default handling.
|
|
71
|
+
export async function lockDataset(datasetId, reason) {
|
|
72
|
+
const exit = await Effect.runPromiseExit(lockRequest(datasetId, reason, datasetTransferTimeoutMs()));
|
|
73
|
+
if (Exit.isSuccess(exit))
|
|
74
|
+
return exit.value;
|
|
75
|
+
throw Cause.squash(exit.cause);
|
|
76
|
+
}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import { Schema } from 'effect';
|
|
2
|
+
declare const SaveReply: Schema.Struct<{
|
|
3
|
+
readonly version: Schema.Struct<{
|
|
4
|
+
readonly id: Schema.String;
|
|
5
|
+
readonly dataset_id: Schema.String;
|
|
6
|
+
readonly version_num: Schema.Number;
|
|
7
|
+
readonly created_at: Schema.String;
|
|
8
|
+
}>;
|
|
9
|
+
}>;
|
|
10
|
+
export type SavedReadmeVersion = typeof SaveReply.Type['version'];
|
|
11
|
+
export declare function saveDatasetReadme(datasetId: string, markdown: string): Promise<SavedReadmeVersion>;
|
|
12
|
+
export {};
|
|
13
|
+
//# sourceMappingURL=dataset-readme-workflow.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"dataset-readme-workflow.d.ts","sourceRoot":"","sources":["../src/dataset-readme-workflow.ts"],"names":[],"mappings":"AAAA,OAAO,EAA+B,MAAM,EAAE,MAAM,QAAQ,CAAA;AAW5D,QAAA,MAAM,SAAS;;;;;;;EAOb,CAAA;AAEF,MAAM,MAAM,kBAAkB,GAAG,OAAO,SAAS,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;AAuDjE,wBAAsB,iBAAiB,CAAC,SAAS,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,kBAAkB,CAAC,CAIxG"}
|