orizu 0.5.27 → 0.5.28
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +19 -0
- package/dist/dataset-transfer-download.d.ts +2 -0
- package/dist/dataset-transfer-download.d.ts.map +1 -0
- package/dist/dataset-transfer-download.js +66 -0
- package/dist/dataset-transfer-policy.d.ts +2 -0
- package/dist/dataset-transfer-policy.d.ts.map +1 -0
- package/dist/dataset-transfer-policy.js +8 -0
- package/dist/dataset-transfer-upload.d.ts +42 -0
- package/dist/dataset-transfer-upload.d.ts.map +1 -0
- package/dist/dataset-transfer-upload.js +110 -0
- package/dist/hosted-harness-opencode.d.ts +8 -9
- package/dist/hosted-harness-opencode.d.ts.map +1 -1
- package/dist/hosted-harness-opencode.js +12 -9
- package/dist/hosted-loop.d.ts.map +1 -1
- package/dist/hosted-loop.js +15 -9
- package/dist/http.d.ts +1 -1
- package/dist/http.d.ts.map +1 -1
- package/dist/http.js +10 -4
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +22 -87
- package/package.json +2 -1
- package/src/dataset-transfer-download.ts +61 -0
- package/src/dataset-transfer-policy.ts +8 -0
- package/src/dataset-transfer-upload.ts +145 -0
- package/src/hosted-harness-opencode.ts +12 -9
- package/src/hosted-loop.ts +15 -8
- package/src/http.ts +10 -4
- package/src/index.ts +21 -129
- package/vendor/skills/orizu/references/prompt-control-plane.md +4 -3
package/dist/index.js
CHANGED
|
@@ -16,7 +16,6 @@ import { parseDatasetFile } from './file-parser.js';
|
|
|
16
16
|
import { streamJsonlRowChunks } from './jsonl-stream.js';
|
|
17
17
|
import { parseDatasetReference } from './dataset-download.js';
|
|
18
18
|
import { editDatasetRows as runEditDatasetRows } from './dataset-edit-rows.js';
|
|
19
|
-
import { ensureDatasetUploadSnapshot } from './dataset-upload-snapshot.js';
|
|
20
19
|
import { extractErrorMessage } from './error-response.js';
|
|
21
20
|
import { describeLogoutHttpFailure, describeLogoutTransportFailure, } from './logout-diagnostic.js';
|
|
22
21
|
import { parseGlobalFlags } from './global-flags.js';
|
|
@@ -3931,88 +3930,22 @@ async function changeTeamMemberRole() {
|
|
|
3931
3930
|
}
|
|
3932
3931
|
printLine(`Updated ${sanitizeTerminalText(member.email)} role to ${sanitizeTerminalText(role)}`);
|
|
3933
3932
|
}
|
|
3934
|
-
async function createDatasetFromRows(project, name, sourceType, rows, readmeMarkdown = null) {
|
|
3935
|
-
const response = await authedFetch('/api/cli/datasets/upload', {
|
|
3936
|
-
method: 'POST',
|
|
3937
|
-
headers: {
|
|
3938
|
-
'Content-Type': 'application/json',
|
|
3939
|
-
},
|
|
3940
|
-
body: JSON.stringify({
|
|
3941
|
-
projectSlug: project,
|
|
3942
|
-
name,
|
|
3943
|
-
rows,
|
|
3944
|
-
sourceType,
|
|
3945
|
-
...(readmeMarkdown !== null ? { readmeMarkdown } : {}),
|
|
3946
|
-
}),
|
|
3947
|
-
});
|
|
3948
|
-
if (!response.ok) {
|
|
3949
|
-
const body = await parseJsonResponse(response, 'Dataset upload');
|
|
3950
|
-
throw new Error(`Upload failed: ${body.error}`);
|
|
3951
|
-
}
|
|
3952
|
-
return parseJsonResponse(response, 'Dataset upload');
|
|
3953
|
-
}
|
|
3954
3933
|
async function uploadJsonlDatasetInChunks(file, project, datasetName, readmeMarkdown = null) {
|
|
3955
|
-
|
|
3956
|
-
|
|
3957
|
-
|
|
3958
|
-
|
|
3959
|
-
|
|
3960
|
-
|
|
3961
|
-
|
|
3962
|
-
|
|
3963
|
-
|
|
3964
|
-
catch (error) {
|
|
3965
|
-
const message = error instanceof Error ? error.message : String(error);
|
|
3966
|
-
if (!dataset) {
|
|
3967
|
-
throw new Error(message);
|
|
3968
|
-
}
|
|
3969
|
-
throw new Error(`Upload stopped while reading the next JSONL chunk: ${message}\n` +
|
|
3970
|
-
`Dataset ${sanitizeTerminalText(dataset.name)} (${sanitizeTerminalText(dataset.id)}) was created and ${totalUploaded} rows were uploaded. ` +
|
|
3971
|
-
`Fix the file, remove the first ${totalUploaded} rows, and run ` +
|
|
3972
|
-
`orizu datasets append --dataset ${dataset.id} --file <remaining-file>.`);
|
|
3973
|
-
}
|
|
3974
|
-
if (nextChunk.done) {
|
|
3975
|
-
break;
|
|
3976
|
-
}
|
|
3977
|
-
const chunk = nextChunk.value;
|
|
3978
|
-
chunkIndex += 1;
|
|
3979
|
-
printLine(`Uploading chunk ${chunkIndex} (${chunk.length} rows)...`);
|
|
3980
|
-
try {
|
|
3981
|
-
if (!dataset) {
|
|
3982
|
-
const data = await createDatasetFromRows(project, datasetName, 'jsonl', chunk, readmeMarkdown);
|
|
3983
|
-
dataset = data.dataset;
|
|
3984
|
-
totalUploaded = data.dataset.rowCount;
|
|
3985
|
-
continue;
|
|
3986
|
-
}
|
|
3987
|
-
const data = await appendChunk(dataset.id, chunk);
|
|
3988
|
-
totalUploaded += data.appendedCount;
|
|
3989
|
-
dataset = {
|
|
3990
|
-
...dataset,
|
|
3991
|
-
rowCount: data.dataset.rowCount,
|
|
3992
|
-
};
|
|
3993
|
-
}
|
|
3994
|
-
catch (error) {
|
|
3995
|
-
const message = error instanceof Error ? error.message : String(error);
|
|
3996
|
-
if (!dataset) {
|
|
3997
|
-
throw new Error(message);
|
|
3998
|
-
}
|
|
3999
|
-
throw new Error(`Chunk ${chunkIndex} failed: ${message}\n` +
|
|
4000
|
-
`Dataset ${dataset.name} (${dataset.id}) was created and ${totalUploaded} rows were uploaded. ` +
|
|
4001
|
-
`To retry, remove the first ${totalUploaded} rows from your file and run ` +
|
|
4002
|
-
`orizu datasets append --dataset ${dataset.id} --file <remaining-file>.`);
|
|
4003
|
-
}
|
|
4004
|
-
}
|
|
4005
|
-
if (!dataset) {
|
|
4006
|
-
throw new Error('Dataset file contains no rows');
|
|
4007
|
-
}
|
|
4008
|
-
const version = await ensureDatasetUploadSnapshot(dataset, hasJsonFlag(), requestDatasetVersionSnapshot);
|
|
3934
|
+
const { transferDatasetUpload } = await import('./dataset-transfer-upload.js');
|
|
3935
|
+
const { data, version } = await transferDatasetUpload({
|
|
3936
|
+
project, name: datasetName, source: { kind: 'jsonl', file }, readmeMarkdown,
|
|
3937
|
+
json: hasJsonFlag(),
|
|
3938
|
+
// JSON mode reserves stderr for structured recovery failures as well.
|
|
3939
|
+
onChunk: (index, count) => { if (!hasJsonFlag())
|
|
3940
|
+
printLine(`Uploading chunk ${index} (${count} rows)...`); },
|
|
3941
|
+
});
|
|
3942
|
+
const dataset = data.dataset;
|
|
4009
3943
|
if (hasJsonFlag())
|
|
4010
3944
|
return printJson({ dataset_id: dataset.id, dataset, dataset_version_id: version.datasetVersion.id });
|
|
4011
3945
|
printLine(`Uploaded dataset ${sanitizeTerminalText(dataset.name)} (${sanitizeTerminalText(dataset.id)}) with ${dataset.rowCount} rows.`);
|
|
4012
3946
|
printLine(`Created dataset version ${sanitizeTerminalText(version.datasetVersion.id)}.`);
|
|
4013
|
-
if (dataset.url)
|
|
3947
|
+
if (dataset.url)
|
|
4014
3948
|
printLine(`View dataset: ${formatTerminalLink(dataset.url)}`);
|
|
4015
|
-
}
|
|
4016
3949
|
}
|
|
4017
3950
|
async function uploadDataset() {
|
|
4018
3951
|
const projectArg = getArg('--project');
|
|
@@ -4031,8 +3964,11 @@ async function uploadDataset() {
|
|
|
4031
3964
|
return;
|
|
4032
3965
|
}
|
|
4033
3966
|
const { rows, sourceType } = parseDatasetFile(file);
|
|
4034
|
-
const
|
|
4035
|
-
const version = await
|
|
3967
|
+
const { transferDatasetUpload } = await import('./dataset-transfer-upload.js');
|
|
3968
|
+
const { data, version } = await transferDatasetUpload({
|
|
3969
|
+
project, name: datasetName, source: { kind: 'rows', rows, sourceType }, readmeMarkdown,
|
|
3970
|
+
json: hasJsonFlag(), onChunk: () => { },
|
|
3971
|
+
});
|
|
4036
3972
|
if (hasJsonFlag())
|
|
4037
3973
|
return printJson({ dataset_id: data.dataset.id, dataset: data.dataset, dataset_version_id: version.datasetVersion.id });
|
|
4038
3974
|
printLine(`Uploaded dataset ${sanitizeTerminalText(data.dataset.name)} (${sanitizeTerminalText(data.dataset.id)}) with ${data.dataset.rowCount} rows.`);
|
|
@@ -4054,8 +3990,11 @@ async function pushDataset() {
|
|
|
4054
3990
|
const project = await resolveProjectSlug(projectArg);
|
|
4055
3991
|
const datasetName = name || basename(file);
|
|
4056
3992
|
const { rows, sourceType } = parseDatasetFile(file);
|
|
4057
|
-
const
|
|
4058
|
-
const version = await
|
|
3993
|
+
const { transferDatasetUpload } = await import('./dataset-transfer-upload.js');
|
|
3994
|
+
const { data, version } = await transferDatasetUpload({
|
|
3995
|
+
project, name: datasetName, source: { kind: 'rows', rows, sourceType }, readmeMarkdown,
|
|
3996
|
+
json: hasJsonFlag(), onChunk: () => { },
|
|
3997
|
+
});
|
|
4059
3998
|
if (hasJsonFlag()) {
|
|
4060
3999
|
const payload = {
|
|
4061
4000
|
dataset_id: data.dataset.id,
|
|
@@ -4134,15 +4073,11 @@ async function downloadDataset() {
|
|
|
4134
4073
|
const selected = await selectDatasetInteractively(projectArg);
|
|
4135
4074
|
datasetId = selected.datasetId;
|
|
4136
4075
|
}
|
|
4137
|
-
const response = await authedFetch(`/api/cli/datasets/${encodeURIComponent(datasetId)}/download?format=${encodeURIComponent(format)}`);
|
|
4138
|
-
if (!response.ok) {
|
|
4139
|
-
throw new Error(`Download failed: ${await response.text()}`);
|
|
4140
|
-
}
|
|
4141
4076
|
const filename = outPathArg
|
|
4142
4077
|
? expandHomePath(outPathArg)
|
|
4143
4078
|
: `${datasetId}.${format}`;
|
|
4144
|
-
const
|
|
4145
|
-
|
|
4079
|
+
const { transferDatasetDownload } = await import('./dataset-transfer-download.js');
|
|
4080
|
+
await transferDatasetDownload(datasetId, format, filename);
|
|
4146
4081
|
if (hasJsonFlag()) {
|
|
4147
4082
|
printJson({ datasetId, format, savedTo: filename });
|
|
4148
4083
|
return;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "orizu",
|
|
3
|
-
"version": "0.5.
|
|
3
|
+
"version": "0.5.28",
|
|
4
4
|
"private": false,
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "./dist/index.js",
|
|
@@ -35,6 +35,7 @@
|
|
|
35
35
|
"dependencies": {
|
|
36
36
|
"@anthropic-ai/claude-agent-sdk": "0.3.201",
|
|
37
37
|
"@vercel/sandbox": "1.10.2",
|
|
38
|
+
"effect": "3.22.1",
|
|
38
39
|
"esbuild": "^0.25.8"
|
|
39
40
|
},
|
|
40
41
|
"devDependencies": {
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
import { Effect, Either, Schedule } from 'effect'
|
|
2
|
+
import { writeFileSync } from 'node:fs'
|
|
3
|
+
import { datasetTransferTimeoutMs } from './dataset-transfer-policy.js'
|
|
4
|
+
import { authedFetch } from './http.js'
|
|
5
|
+
|
|
6
|
+
class DownloadHttpError extends Error {
|
|
7
|
+
readonly _tag = 'DownloadHttpError'
|
|
8
|
+
constructor(readonly status: number, detail: string) {
|
|
9
|
+
super(`Download failed (HTTP ${status}): ${detail}`)
|
|
10
|
+
}
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
class DownloadTransportError extends Error {
|
|
14
|
+
readonly _tag = 'DownloadTransportError'
|
|
15
|
+
constructor(cause: unknown) {
|
|
16
|
+
super('Dataset download transport failed', { cause })
|
|
17
|
+
}
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
export async function transferDatasetDownload(datasetId: string, format: string, filename: string): Promise<void> {
|
|
21
|
+
const timeoutMs = datasetTransferTimeoutMs()
|
|
22
|
+
const download = Effect.tryPromise({
|
|
23
|
+
try: async signal => {
|
|
24
|
+
const response = await authedFetch(`/api/cli/datasets/${encodeURIComponent(datasetId)}/download?format=${encodeURIComponent(format)}`, { signal })
|
|
25
|
+
if (!response.ok) throw new DownloadHttpError(response.status, await response.text())
|
|
26
|
+
return new Uint8Array(await response.arrayBuffer())
|
|
27
|
+
},
|
|
28
|
+
catch: cause => {
|
|
29
|
+
if (cause instanceof TypeError) return new DownloadTransportError(cause)
|
|
30
|
+
return cause instanceof Error ? cause : new Error('Dataset download request failed', { cause })
|
|
31
|
+
},
|
|
32
|
+
}).pipe(Effect.timeoutFail({
|
|
33
|
+
duration: timeoutMs,
|
|
34
|
+
onTimeout: () => new Error('DATASET_TRANSFER_TIMEOUT: Dataset download exceeded its deadline'),
|
|
35
|
+
}), Effect.retry({
|
|
36
|
+
times: 2,
|
|
37
|
+
schedule: Schedule.spaced(100),
|
|
38
|
+
while: error => error instanceof DownloadTransportError || (error instanceof DownloadHttpError && [429, 502, 503, 504].includes(error.status)),
|
|
39
|
+
}))
|
|
40
|
+
const fileError = (cause: unknown) => new Error(`DATASET_DOWNLOAD_FILE: Could not save dataset to ${filename}: ${cause instanceof Error ? cause.message : String(cause)}`, { cause })
|
|
41
|
+
const write = (bytes: Uint8Array) => Effect.try({
|
|
42
|
+
// Wait for the complete response, then retain the CLI's existing in-place
|
|
43
|
+
// write semantics, including inode metadata, hardlinks and symlink targets.
|
|
44
|
+
try: () => writeFileSync(filename, bytes, { mode: 0o600 }),
|
|
45
|
+
catch: fileError,
|
|
46
|
+
})
|
|
47
|
+
const controller = new AbortController()
|
|
48
|
+
const handleInterrupt = () => controller.abort()
|
|
49
|
+
process.once('SIGINT', handleInterrupt)
|
|
50
|
+
process.once('SIGTERM', handleInterrupt)
|
|
51
|
+
try {
|
|
52
|
+
const result = await Effect.runPromise(Effect.either(Effect.flatMap(download, write)), { signal: controller.signal })
|
|
53
|
+
if (Either.isLeft(result)) throw result.left
|
|
54
|
+
} catch (error) {
|
|
55
|
+
if (controller.signal.aborted) throw new Error('DATASET_TRANSFER_INTERRUPTED: Dataset download cancelled')
|
|
56
|
+
throw error
|
|
57
|
+
} finally {
|
|
58
|
+
process.removeListener('SIGINT', handleInterrupt)
|
|
59
|
+
process.removeListener('SIGTERM', handleInterrupt)
|
|
60
|
+
}
|
|
61
|
+
}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
export function datasetTransferTimeoutMs(): number {
|
|
2
|
+
const configured = process.env.ORIZU_DATASET_TRANSFER_TIMEOUT_MS
|
|
3
|
+
const timeoutMs = configured === undefined ? 120_000 : Number(configured)
|
|
4
|
+
if (!Number.isSafeInteger(timeoutMs) || timeoutMs < 1 || timeoutMs > 300_000) {
|
|
5
|
+
throw new Error('DATASET_TRANSFER_CONFIG: ORIZU_DATASET_TRANSFER_TIMEOUT_MS must be an integer from 1 to 300000')
|
|
6
|
+
}
|
|
7
|
+
return timeoutMs
|
|
8
|
+
}
|
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
import { Context, Effect, Either, Layer } from 'effect'
|
|
2
|
+
|
|
3
|
+
import { ensureDatasetUploadSnapshot } from './dataset-upload-snapshot.js'
|
|
4
|
+
import { datasetTransferTimeoutMs } from './dataset-transfer-policy.js'
|
|
5
|
+
import { extractErrorMessage } from './error-response.js'
|
|
6
|
+
import { authedFetch } from './http.js'
|
|
7
|
+
import { parseJsonResponse, sanitizeTerminalText } from './json-response.js'
|
|
8
|
+
import { streamJsonlRowChunks } from './jsonl-stream.js'
|
|
9
|
+
|
|
10
|
+
export interface DatasetUploadResponse {
|
|
11
|
+
dataset: { id: string; name: string; rowCount: number; sourceType: string; url?: string }
|
|
12
|
+
readmeVersion?: { id: string; version_num: number; created_at: string } | null
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
interface DatasetSnapshot { datasetVersion: { id: string } }
|
|
16
|
+
interface DatasetAppendResponse { dataset: { id: string; name: string; rowCount: number }; appendedCount: number }
|
|
17
|
+
type UploadRows = Array<Record<string, unknown>>
|
|
18
|
+
type UploadSource = { kind: 'jsonl'; file: string } | { kind: 'rows'; rows: UploadRows; sourceType: 'csv' | 'json' | 'jsonl' }
|
|
19
|
+
|
|
20
|
+
export interface DatasetUploadOptions {
|
|
21
|
+
project: string
|
|
22
|
+
name: string
|
|
23
|
+
source: UploadSource
|
|
24
|
+
readmeMarkdown: string | null
|
|
25
|
+
json: boolean
|
|
26
|
+
onChunk: (index: number, rowCount: number) => void
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
class DatasetWriteFailure extends Error {
|
|
30
|
+
readonly _tag = 'DatasetWriteFailure'
|
|
31
|
+
constructor(message: string, readonly outcome: 'rejected' | 'unknown', options?: ErrorOptions) {
|
|
32
|
+
super(message, options)
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
class DatasetInputFailure extends Error {
|
|
37
|
+
readonly _tag = 'DatasetInputFailure'
|
|
38
|
+
constructor(cause: unknown) { super(cause instanceof Error ? cause.message : String(cause), { cause }) }
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
interface DatasetUploadApi {
|
|
42
|
+
create: (rows: UploadRows, sourceType: string) => Effect.Effect<DatasetUploadResponse, DatasetWriteFailure>
|
|
43
|
+
append: (id: string, rows: UploadRows) => Effect.Effect<DatasetAppendResponse, DatasetWriteFailure>
|
|
44
|
+
snapshot: (id: string) => Effect.Effect<DatasetSnapshot, DatasetWriteFailure>
|
|
45
|
+
}
|
|
46
|
+
const DatasetUploadApi = Context.GenericTag<DatasetUploadApi>('orizu/DatasetUploadApi')
|
|
47
|
+
|
|
48
|
+
// Only the existing authentication client's rejected-token refresh may replay a
|
|
49
|
+
// request. No transient/transport retries belong around these non-idempotent writes.
|
|
50
|
+
function post<T>(path: string, body: unknown, context: string, prefix: string, timeoutMs: number): Effect.Effect<T, DatasetWriteFailure> {
|
|
51
|
+
return Effect.suspend(() => {
|
|
52
|
+
let requestStarted = false
|
|
53
|
+
return Effect.tryPromise({
|
|
54
|
+
try: async signal => {
|
|
55
|
+
const response = await authedFetch(path, {
|
|
56
|
+
method: 'POST', headers: { 'Content-Type': 'application/json' },
|
|
57
|
+
body: JSON.stringify(body), signal,
|
|
58
|
+
}, () => { requestStarted = true })
|
|
59
|
+
if (!response.ok) {
|
|
60
|
+
throw new DatasetWriteFailure(`${prefix}: ${await extractErrorMessage(response)}`, response.status >= 500 ? 'unknown' : 'rejected')
|
|
61
|
+
}
|
|
62
|
+
return await parseJsonResponse<T>(response, context)
|
|
63
|
+
},
|
|
64
|
+
catch: cause => cause instanceof DatasetWriteFailure ? cause : new DatasetWriteFailure(
|
|
65
|
+
cause instanceof Error ? cause.message : String(cause), requestStarted ? 'unknown' : 'rejected', { cause },
|
|
66
|
+
),
|
|
67
|
+
}).pipe(Effect.timeoutFail({
|
|
68
|
+
duration: timeoutMs,
|
|
69
|
+
onTimeout: () => new DatasetWriteFailure(`DATASET_TRANSFER_TIMEOUT: ${context} exceeded its deadline`, requestStarted ? 'unknown' : 'rejected'),
|
|
70
|
+
}))
|
|
71
|
+
})
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
function liveUploadApi(options: DatasetUploadOptions) {
|
|
75
|
+
const timeoutMs = datasetTransferTimeoutMs()
|
|
76
|
+
return Layer.succeed(DatasetUploadApi, {
|
|
77
|
+
create: (rows, sourceType) => post<DatasetUploadResponse>('/api/cli/datasets/upload', {
|
|
78
|
+
projectSlug: options.project, name: options.name, rows, sourceType,
|
|
79
|
+
...(options.readmeMarkdown !== null ? { readmeMarkdown: options.readmeMarkdown } : {}),
|
|
80
|
+
}, 'Dataset upload', 'Upload failed', timeoutMs),
|
|
81
|
+
append: (id, rows) => post<DatasetAppendResponse>(`/api/cli/datasets/${encodeURIComponent(id)}/rows`, { rows }, 'Dataset append', 'Append failed', timeoutMs),
|
|
82
|
+
snapshot: id => post<DatasetSnapshot>(`/api/cli/datasets/${encodeURIComponent(id)}/versions`, {}, 'Dataset version create', 'Failed to create dataset version', timeoutMs),
|
|
83
|
+
})
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
async function expectedPromise<A, E>(program: Effect.Effect<A, E>): Promise<A> {
|
|
87
|
+
const result = await Effect.runPromise(Effect.either(program))
|
|
88
|
+
if (Either.isLeft(result)) throw result.left
|
|
89
|
+
return result.right
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
function uploadProgram(options: DatasetUploadOptions) {
|
|
93
|
+
return Effect.gen(function* () {
|
|
94
|
+
const api = yield* DatasetUploadApi
|
|
95
|
+
const chunks = yield* Effect.acquireRelease(
|
|
96
|
+
Effect.sync(() => options.source.kind === 'jsonl'
|
|
97
|
+
? streamJsonlRowChunks(options.source.file)
|
|
98
|
+
: (async function* () { if (options.source.kind === 'rows') yield options.source.rows })()),
|
|
99
|
+
iterator => Effect.promise(async () => { await iterator.return(undefined) }),
|
|
100
|
+
)
|
|
101
|
+
let data: DatasetUploadResponse | undefined
|
|
102
|
+
let confirmedRows = 0
|
|
103
|
+
let chunkIndex = 0
|
|
104
|
+
while (true) {
|
|
105
|
+
const next = yield* Effect.tryPromise({ try: () => chunks.next(), catch: cause => new DatasetInputFailure(cause) }).pipe(
|
|
106
|
+
Effect.mapError(error => new Error(data
|
|
107
|
+
? `Upload stopped while reading the next JSONL chunk: ${error.message}\nDataset ${sanitizeTerminalText(data.dataset.name)} (${sanitizeTerminalText(data.dataset.id)}) was created and ${confirmedRows} rows were uploaded. Fix the file, remove the first ${confirmedRows} rows, and run orizu datasets append --dataset ${data.dataset.id} --file <remaining-file>.`
|
|
108
|
+
: error.message)),
|
|
109
|
+
)
|
|
110
|
+
if (next.done) break
|
|
111
|
+
chunkIndex++
|
|
112
|
+
if (options.source.kind === 'jsonl') yield* Effect.sync(() => options.onChunk(chunkIndex, next.value.length))
|
|
113
|
+
const write = data
|
|
114
|
+
? api.append(data.dataset.id, next.value).pipe(Effect.map(result => ({
|
|
115
|
+
data: { ...data!, dataset: { ...data!.dataset, rowCount: result.dataset.rowCount } },
|
|
116
|
+
added: result.appendedCount,
|
|
117
|
+
})))
|
|
118
|
+
: api.create(next.value, options.source.kind === 'jsonl' ? 'jsonl' : options.source.sourceType).pipe(
|
|
119
|
+
Effect.map(result => ({ data: result, added: result.dataset.rowCount })),
|
|
120
|
+
)
|
|
121
|
+
const result = yield* write.pipe(Effect.mapError(error => {
|
|
122
|
+
if (error.outcome === 'unknown') {
|
|
123
|
+
return new Error(`DATASET_UPLOAD_UNCERTAIN: Chunk ${chunkIndex} may have committed, but its outcome could not be confirmed: ${error.message}\n` +
|
|
124
|
+
(data ? `Dataset ${sanitizeTerminalText(data.dataset.id)} has ${confirmedRows} confirmed uploaded rows. ` : 'Dataset creation could not be confirmed. ') +
|
|
125
|
+
'Inspect the dataset and reconcile row IDs before retrying; do not replay this chunk blindly.')
|
|
126
|
+
}
|
|
127
|
+
return new Error(data
|
|
128
|
+
? `Chunk ${chunkIndex} failed: ${error.message}\nDataset ${sanitizeTerminalText(data.dataset.name)} (${sanitizeTerminalText(data.dataset.id)}) was created and ${confirmedRows} rows were uploaded. To retry, remove the first ${confirmedRows} rows from your file and run orizu datasets append --dataset ${data.dataset.id} --file <remaining-file>.`
|
|
129
|
+
: error.message)
|
|
130
|
+
}))
|
|
131
|
+
data = result.data
|
|
132
|
+
confirmedRows += result.added
|
|
133
|
+
}
|
|
134
|
+
if (!data) return yield* Effect.fail(new Error('Dataset file contains no rows'))
|
|
135
|
+
const version = yield* Effect.tryPromise({
|
|
136
|
+
try: () => ensureDatasetUploadSnapshot(data!.dataset, options.json, id => expectedPromise(api.snapshot(id))),
|
|
137
|
+
catch: cause => cause instanceof Error ? cause : new Error(String(cause)),
|
|
138
|
+
})
|
|
139
|
+
return { data, version }
|
|
140
|
+
})
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
export function transferDatasetUpload(options: DatasetUploadOptions): Promise<{ data: DatasetUploadResponse; version: DatasetSnapshot }> {
|
|
144
|
+
return expectedPromise(uploadProgram(options).pipe(Effect.scoped, Effect.provide(liveUploadApi(options))))
|
|
145
|
+
}
|
|
@@ -1070,15 +1070,12 @@ export interface OpenCodeConfigOptions {
|
|
|
1070
1070
|
}
|
|
1071
1071
|
|
|
1072
1072
|
/**
|
|
1073
|
-
* Tools
|
|
1074
|
-
*
|
|
1075
|
-
*
|
|
1076
|
-
*
|
|
1077
|
-
*
|
|
1078
|
-
*
|
|
1079
|
-
* to it as a tool error) so it proceeds autonomously per the standing preamble
|
|
1080
|
-
* (HOSTED_TASK_PREAMBLE), instead of blocking the run forever. This is the
|
|
1081
|
-
* documented fallback path in ALI-1037 (answer-externally is not cleanly possible).
|
|
1073
|
+
* Tools denied for CLI-origin headless runs (ALI-1037). OpenCode 1.14.41 removes
|
|
1074
|
+
* a tool from the model toolset when its permission rule is `deny` on `*`; it does
|
|
1075
|
+
* not expose the tool and report a denial result. Hosted-web therefore overrides
|
|
1076
|
+
* `question` to `allow`, intercepts its measured running part, and quiesces the
|
|
1077
|
+
* turn while waiting for the human. CLI-origin runs retain this deny list because
|
|
1078
|
+
* no human reply flow exists there.
|
|
1082
1079
|
*/
|
|
1083
1080
|
export const HEADLESS_DENIED_TOOLS = ['question'] as const
|
|
1084
1081
|
|
|
@@ -1090,6 +1087,12 @@ export function defaultHeadlessPermission(): Record<string, unknown> {
|
|
|
1090
1087
|
return permission
|
|
1091
1088
|
}
|
|
1092
1089
|
|
|
1090
|
+
/** Select the OpenCode permission map for the session's human-input surface. */
|
|
1091
|
+
export function permissionForOrigin(origin?: 'hosted-web' | 'cli'): Record<string, unknown> {
|
|
1092
|
+
if (origin === 'hosted-web') return { ...defaultHeadlessPermission(), question: 'allow' }
|
|
1093
|
+
return defaultHeadlessPermission()
|
|
1094
|
+
}
|
|
1095
|
+
|
|
1093
1096
|
/**
|
|
1094
1097
|
* Build the `OPENCODE_CONFIG_CONTENT` payload OpenCode reads at boot (mirrors
|
|
1095
1098
|
* OpenInspect's `start_opencode`). Kept pure so the launch command and the
|
package/src/hosted-loop.ts
CHANGED
|
@@ -55,6 +55,7 @@ import {
|
|
|
55
55
|
OPENCODE_PINNED_VERSION,
|
|
56
56
|
awaitOpenCodeModelResolvable,
|
|
57
57
|
createOpenCodeHarness,
|
|
58
|
+
permissionForOrigin,
|
|
58
59
|
spawnOpenCode,
|
|
59
60
|
type ModelValidationOutcome,
|
|
60
61
|
type SpawnOpenCodeOptions,
|
|
@@ -612,6 +613,7 @@ export async function runHostedLoop(opts: RunHostedLoopOptions): Promise<HostedL
|
|
|
612
613
|
// connection-refused "fetch failed" (ALI-1034).
|
|
613
614
|
spawned = await spawnImpl({
|
|
614
615
|
model: context.model,
|
|
616
|
+
permission: permissionForOrigin(context.sessionOrigin),
|
|
615
617
|
cwd: context.workspaceDir,
|
|
616
618
|
port: context.opencodePort,
|
|
617
619
|
env: buildOpenCodeSpawnEnv(context),
|
|
@@ -699,8 +701,9 @@ export async function runHostedLoop(opts: RunHostedLoopOptions): Promise<HostedL
|
|
|
699
701
|
content: composeHostedTaskPrompt(taskPrompt, opts.taskPreamble ?? resolveHostedTaskPreamble(context.sessionOrigin)),
|
|
700
702
|
author: context.author,
|
|
701
703
|
}
|
|
702
|
-
// Drive the prompt through
|
|
703
|
-
//
|
|
704
|
+
// Drive the prompt through question-deny annotation unless hosted-web
|
|
705
|
+
// OpenCode interception owns the outcome, then through the idle watchdog
|
|
706
|
+
// (ALI-1037): no harness event for `idleTimeoutMs` aborts the
|
|
704
707
|
// prompt and throws `AgentStalledError` → the catch below harvests, then
|
|
705
708
|
// finishes the run `failed`. The prompt runs under a CHILD AbortController
|
|
706
709
|
// linked to the loop signal so the watchdog can abort the in-flight prompt
|
|
@@ -742,9 +745,10 @@ export async function runHostedLoop(opts: RunHostedLoopOptions): Promise<HostedL
|
|
|
742
745
|
|
|
743
746
|
const terminalSummary: Record<string, unknown> = { agentSessionId }
|
|
744
747
|
let hasQuestionOutcome = false
|
|
745
|
-
const
|
|
746
|
-
const
|
|
747
|
-
|
|
748
|
+
const promptEvents = harness.runPrompt(prompt, promptController.signal)
|
|
749
|
+
const hasHostedQuestionInterception = harnessKind === 'opencode' && context.sessionOrigin === 'hosted-web'
|
|
750
|
+
const intercepted = hasHostedQuestionInterception
|
|
751
|
+
? interceptHostedQuestions(promptEvents, {
|
|
748
752
|
onDetected: question => {
|
|
749
753
|
hasQuestionOutcome = true
|
|
750
754
|
terminalSummary.outcome = 'question_pending'
|
|
@@ -760,7 +764,10 @@ export async function runHostedLoop(opts: RunHostedLoopOptions): Promise<HostedL
|
|
|
760
764
|
onPersisted: beginQuiescence,
|
|
761
765
|
onInvalidPersisted: beginQuiescence,
|
|
762
766
|
})
|
|
763
|
-
:
|
|
767
|
+
: promptEvents
|
|
768
|
+
const questionAware = hasHostedQuestionInterception
|
|
769
|
+
? intercepted
|
|
770
|
+
: annotateHeadlessQuestions(intercepted)
|
|
764
771
|
const interruptAware = (async function* (): AsyncGenerator<HarnessEvent> {
|
|
765
772
|
for await (const event of questionAware) {
|
|
766
773
|
if (isHarnessTerminalKind(event.kind)) {
|
|
@@ -811,8 +818,8 @@ export async function runHostedLoop(opts: RunHostedLoopOptions): Promise<HostedL
|
|
|
811
818
|
promptController.abort()
|
|
812
819
|
await harness.stop()
|
|
813
820
|
},
|
|
814
|
-
//
|
|
815
|
-
//
|
|
821
|
+
// Without hosted-web OpenCode interception, a model stuck retrying a
|
|
822
|
+
// denied question emits synthetic `question_auto_answered` events. Those are
|
|
816
823
|
// NOT progress — excluding them lets a deny/retry loop trip agent_stalled
|
|
817
824
|
// instead of resetting the idle timer forever (ALI-1069).
|
|
818
825
|
isProgress: event => event.kind !== 'question_auto_answered',
|
package/src/http.ts
CHANGED
|
@@ -422,7 +422,13 @@ export function captureAuthenticatedRequestContext(
|
|
|
422
422
|
}
|
|
423
423
|
}
|
|
424
424
|
|
|
425
|
-
|
|
425
|
+
// The optional observer runs immediately before a resource fetch, after auth
|
|
426
|
+
// preflight. Authentication refresh requests do not mark the resource as sent.
|
|
427
|
+
export async function authedFetch(path: string, init: RequestInit = {}, onRequestStart?: () => void) {
|
|
428
|
+
const fetchResource = (url: string, requestInit: RequestInit) => {
|
|
429
|
+
onRequestStart?.()
|
|
430
|
+
return fetch(url, requestInit)
|
|
431
|
+
}
|
|
426
432
|
const baseUrl = resolveBaseUrl()
|
|
427
433
|
assertSecureTokenTransport(baseUrl)
|
|
428
434
|
|
|
@@ -433,7 +439,7 @@ export async function authedFetch(path: string, init: RequestInit = {}) {
|
|
|
433
439
|
// the file and the next request naturally picks up the new bearer.
|
|
434
440
|
const envBearer = resolveEnvBearerToken()
|
|
435
441
|
if (envBearer) {
|
|
436
|
-
return await
|
|
442
|
+
return await fetchResource(`${baseUrl}${path}`, {
|
|
437
443
|
...init,
|
|
438
444
|
headers: {
|
|
439
445
|
...(init.headers || {}),
|
|
@@ -451,7 +457,7 @@ export async function authedFetch(path: string, init: RequestInit = {}) {
|
|
|
451
457
|
const resolved = await refreshExpiredStoredCredentials(baseUrl, credentials, init.signal)
|
|
452
458
|
let activeCredentials = resolved.credentials
|
|
453
459
|
|
|
454
|
-
let response = await
|
|
460
|
+
let response = await fetchResource(`${baseUrl}${path}`, {
|
|
455
461
|
...init,
|
|
456
462
|
headers: {
|
|
457
463
|
...(init.headers || {}),
|
|
@@ -476,7 +482,7 @@ export async function authedFetch(path: string, init: RequestInit = {}) {
|
|
|
476
482
|
)
|
|
477
483
|
}
|
|
478
484
|
activeCredentials = requireCurrentCredentials(baseUrl, refreshResult.credentials)
|
|
479
|
-
response = await
|
|
485
|
+
response = await fetchResource(`${baseUrl}${path}`, {
|
|
480
486
|
...init,
|
|
481
487
|
headers: {
|
|
482
488
|
...(init.headers || {}),
|