orizu 0.5.26 → 0.5.28

package/README.md

@@ -427,6 +427,25 @@ Task and dataset export support:
 - `json`
 - `jsonl`
 
+Dataset `upload` and `push` create an immutable version after the rows are
+acknowledged. With `--json`, upload prints one result document. A failed snapshot
+reports the uploaded dataset identity so recovery can target the snapshot step.
+If a write's response is lost or times out, inspect the dataset before retrying:
+the last chunk may already have committed.
+
+Dataset downloads retry transient read failures up to three attempts. Each
+transfer request, including its response body, has a two-minute deadline;
+`ORIZU_DATASET_TRANSFER_TIMEOUT_MS` accepts 1–300000 milliseconds. Downloads
+receive the complete response before writing to the destination. Existing files
+keep their inode, ownership, permissions and ACLs; writes follow existing
+symlinks and update hardlinked readers. New files request mode `0600`, subject
+to the filesystem's access rules. A failed network transfer leaves an existing output
+untouched. The final write retains ordinary filesystem behavior: it is not
+atomic, and a disk failure during that write can leave partial output.
+SIGINT/SIGTERM cancels a download. Upload writes
+have no additional transient retries; the existing session refresh behavior is
+unchanged.
+
 ## Interactive And Automated Usage
 
 Many commands can prompt for missing team, project, app, dataset, or task selections in an interactive terminal. In scripts and CI, pass explicit flags instead:

package/dist/artifacts-git-runtime.d.ts

@@ -0,0 +1,34 @@
+export interface GitCommandResult {
+    exitCode: number;
+    stdout: string;
+    stderr: string;
+    timedOut?: boolean;
+    cancelled?: boolean;
+}
+export interface GitAuth {
+    askPassPath: string;
+    token: string;
+}
+export interface GitRunOptions {
+    cwd: string;
+    auth?: GitAuth;
+    env?: NodeJS.ProcessEnv;
+    timeoutMs?: number;
+    signal?: AbortSignal;
+    commitIdentityEnv?: Readonly<Record<string, string>>;
+}
+export type GitRunner = (args: string[], options: GitRunOptions) => Promise<GitCommandResult>;
+export declare function assertCommitIdentityEnv(overlay: Readonly<Record<string, string>>): void;
+export declare function parseArtifactTokenExpiry(token: string): number | null;
+export declare function credentialSecretVariants(secrets: readonly (string | undefined)[]): string[];
+export declare function redactSecrets(value: string, secrets: readonly (string | undefined)[]): string;
+export declare function createEphemeralAskPass(root: string): Promise<{
+    path: string;
+    dispose: () => Promise<void>;
+}>;
+export declare function buildGitAuthEnvironment(auth: GitAuth, base?: NodeJS.ProcessEnv): NodeJS.ProcessEnv;
+export declare function assertSafeGitInvocation(args: readonly string[], secrets?: readonly string[]): void;
+export declare const defaultGitRunner: GitRunner;
+export declare function gitOk(runGit: GitRunner, args: string[], options: GitRunOptions, operation: string): Promise<string>;
+export declare function revParse(runGit: GitRunner, cwd: string, ref: string): Promise<string>;
+//# sourceMappingURL=artifacts-git-runtime.d.ts.map

package/dist/artifacts-git-runtime.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"artifacts-git-runtime.d.ts","sourceRoot":"","sources":["../src/artifacts-git-runtime.ts"],"names":[],"mappings":"AAoCA,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,EAAE,MAAM,CAAA;IAChB,MAAM,EAAE,MAAM,CAAA;IACd,MAAM,EAAE,MAAM,CAAA;IACd,QAAQ,CAAC,EAAE,OAAO,CAAA;IAClB,SAAS,CAAC,EAAE,OAAO,CAAA;CACpB;AAED,MAAM,WAAW,OAAO;IACtB,WAAW,EAAE,MAAM,CAAA;IACnB,KAAK,EAAE,MAAM,CAAA;CACd;AAED,MAAM,WAAW,aAAa;IAC5B,GAAG,EAAE,MAAM,CAAA;IACX,IAAI,CAAC,EAAE,OAAO,CAAA;IACd,GAAG,CAAC,EAAE,MAAM,CAAC,UAAU,CAAA;IACvB,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,MAAM,CAAC,EAAE,WAAW,CAAA;IACpB,iBAAiB,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAA;CACrD;AAED,MAAM,MAAM,SAAS,GAAG,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,OAAO,EAAE,aAAa,KAAK,OAAO,CAAC,gBAAgB,CAAC,CAAA;AAE7F,wBAAgB,uBAAuB,CAAC,OAAO,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,GAAG,IAAI,CASvF;AAED,wBAAgB,wBAAwB,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAkBrE;AAED,wBAAgB,wBAAwB,CAAC,OAAO,EAAE,SAAS,CAAC,MAAM,GAAG,SAAS,CAAC,EAAE,GAAG,MAAM,EAAE,CAS3F;AAED,wBAAgB,aAAa,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,SAAS,CAAC,MAAM,GAAG,SAAS,CAAC,EAAE,GAAG,MAAM,CAK7F;AAED,wBAAsB,sBAAsB,CAC1C,IAAI,EAAE,MAAM,GACX,OAAO,CAAC;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAA;CAAE,CAAC,CAsBzD;AAUD,wBAAgB,uBAAuB,CACrC,IAAI,EAAE,OAAO,EACb,IAAI,GAAE,MAAM,CAAC,UAAwB,GACpC,MAAM,CAAC,UAAU,CAkBnB;AAgBD,wBAAgB,uBAAuB,CACrC,IAAI,EAAE,SAAS,MAAM,EAAE,EACvB,OAAO,GAAE,SAAS,MAAM,EAAO,GAC9B,IAAI,CAYN;AAWD,eAAO,MAAM,gBAAgB,EAAE,SAoI3B,CAAA;AAEJ,wBAAsB,KAAK,CACzB,MAAM,EAAE,SAAS,EACjB,IAAI,EAAE,MAAM,EAAE,EACd,OAAO,EAAE,aAAa,EACtB,SAAS,EAAE,MAAM,GAChB,OAAO,CAAC,MAAM,CAAC,CAUjB;AAED,wBAAsB,QAAQ,CAAC,MAAM,EAAE,SAAS,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAE3F"}

package/dist/artifacts-git-runtime.js

@@ -0,0 +1,312 @@
+import { spawn } from 'node:child_process';
+import { chmod, mkdir, rm, writeFile } from 'node:fs/promises';
+import { join } from 'node:path';
+const ARTIFACT_TOKEN_BODY = /^[^\s\u0000-\u001f\u007f]{16,1024}$/;
+const ARTIFACT_TOKEN_ADVISORY_SUFFIX = '?expires=';
+const ARTIFACT_TOKEN_BARE_BODY = /art_v1_[0-9a-fA-F]{40}/;
+const DEFAULT_GIT_TIMEOUT_MS = 30_000;
+const GIT_TERMINATION_GRACE_MS = 1_000;
+const GIT_CONTEXT_ENV_KEYS = [
+    'GIT_DIR',
+    'GIT_WORK_TREE',
+    'GIT_INDEX_FILE',
+    'GIT_OBJECT_DIRECTORY',
+    'GIT_COMMON_DIR',
+    'GIT_NAMESPACE',
+];
+const GIT_CHILD_ENV_ALLOWLIST = [
+    'PATH',
+    'TMPDIR',
+    'TMP',
+    'TEMP',
+    'SYSTEMROOT',
+    'WINDIR',
+    'COMSPEC',
+    'PATHEXT',
+    'LANG',
+    'TZ',
+    'SSL_CERT_FILE',
+    'SSL_CERT_DIR',
+    'CURL_CA_BUNDLE',
+    'GIT_SSL_CAINFO',
+];
+const COMMIT_IDENTITY_ENV_PATTERN = /^GIT_(AUTHOR|COMMITTER)_(NAME|EMAIL|DATE)$/;
+export function assertCommitIdentityEnv(overlay) {
+    for (const key of Object.keys(overlay)) {
+        if (!COMMIT_IDENTITY_ENV_PATTERN.test(key)) {
+            throw new Error(`Refusing Git environment overlay ${JSON.stringify(key)}: only ` +
+                'GIT_AUTHOR_/GIT_COMMITTER_ NAME, EMAIL and DATE may be set');
+        }
+    }
+}
+export function parseArtifactTokenExpiry(token) {
+    const separator = token.lastIndexOf(ARTIFACT_TOKEN_ADVISORY_SUFFIX);
+    if (separator !== -1) {
+        const body = token.slice(0, separator);
+        const suffix = token.slice(separator + ARTIFACT_TOKEN_ADVISORY_SUFFIX.length);
+        if (!ARTIFACT_TOKEN_BODY.test(body) || !/^\d{1,15}$/.test(suffix)) {
+            throw new Error('Artifacts returned an unexpected token format');
+        }
+        const expiry = Number(suffix);
+        if (!Number.isSafeInteger(expiry)) {
+            throw new Error('Artifacts returned an invalid token expiry');
+        }
+        return expiry;
+    }
+    if (!ARTIFACT_TOKEN_BODY.test(token)) {
+        throw new Error('Artifacts returned an unexpected token format');
+    }
+    return null;
+}
+export function credentialSecretVariants(secrets) {
+    const variants = new Set();
+    for (const secret of secrets) {
+        if (!secret)
+            continue;
+        variants.add(secret);
+        const bareBody = ARTIFACT_TOKEN_BARE_BODY.exec(secret)?.[0];
+        if (bareBody)
+            variants.add(bareBody);
+    }
+    return [...variants];
+}
+export function redactSecrets(value, secrets) {
+    return credentialSecretVariants(secrets)
+        .filter((secret) => secret.length >= 4)
+        .sort((left, right) => right.length - left.length)
+        .reduce((redacted, secret) => redacted.split(secret).join('[REDACTED]'), value);
+}
+export async function createEphemeralAskPass(root) {
+    const authDir = join(root, 'git-auth');
+    await mkdir(authDir, { recursive: true, mode: 0o700 });
+    await chmod(authDir, 0o700);
+    const path = join(authDir, 'askpass.sh');
+    const script = `#!/bin/sh
+case "$1" in
+  *sername*) printf '%s\\n' x ;;
+  *assword*)
+    IFS= read -r password <&3 || exit 1
+    [ -n "$password" ] || exit 1
+    printf '%s\\n' "$password"
+    ;;
+  *) exit 1 ;;
+esac
+`;
+    await writeFile(path, script, { mode: 0o700 });
+    await chmod(path, 0o700);
+    return {
+        path,
+        dispose: () => rm(authDir, { recursive: true, force: true }),
+    };
+}
+function minimalGitChildEnvironment(base) {
+    const selected = { NODE_ENV: base.NODE_ENV };
+    for (const key of GIT_CHILD_ENV_ALLOWLIST) {
+        if (base[key] !== undefined)
+            selected[key] = base[key];
+    }
+    return selected;
+}
+export function buildGitAuthEnvironment(auth, base = process.env) {
+    parseArtifactTokenExpiry(auth.token);
+    const env = {
+        ...minimalGitChildEnvironment(base),
+        LC_ALL: 'C',
+        GIT_TERMINAL_PROMPT: '0',
+        GIT_ASKPASS: auth.askPassPath,
+        GIT_ASKPASS_REQUIRE: 'force',
+        GIT_CONFIG_NOSYSTEM: '1',
+        GIT_CONFIG_GLOBAL: '/dev/null',
+        GIT_CONFIG_COUNT: '2',
+        GIT_CONFIG_KEY_0: 'credential.helper',
+        GIT_CONFIG_VALUE_0: '',
+        GIT_CONFIG_KEY_1: 'credential.useHttpPath',
+        GIT_CONFIG_VALUE_1: 'true',
+    };
+    for (const key of GIT_CONTEXT_ENV_KEYS)
+        delete env[key];
+    return env;
+}
+function buildUnauthenticatedGitEnvironment(base = process.env) {
+    const env = {
+        ...minimalGitChildEnvironment(base),
+        LC_ALL: 'C',
+        GIT_TERMINAL_PROMPT: '0',
+        GIT_CONFIG_NOSYSTEM: '1',
+        GIT_CONFIG_GLOBAL: '/dev/null',
+    };
+    for (const key of GIT_CONTEXT_ENV_KEYS)
+        delete env[key];
+    return env;
+}
+export function assertSafeGitInvocation(args, secrets = []) {
+    const joined = args.join('\u0000');
+    for (const secret of credentialSecretVariants(secrets)) {
+        if (joined.includes(secret))
+            throw new Error('Refusing to place a credential in Git argv');
+    }
+    for (const arg of args) {
+        if (!/^https?:\/\//i.test(arg))
+            continue;
+        const url = new URL(arg);
+        if (url.username || url.password) {
+            throw new Error('Refusing a Git remote URL containing userinfo');
+        }
+    }
+}
+function signalGitProcess(child, signal) {
+    try {
+        if (process.platform !== 'win32' && child.pid)
+            process.kill(-child.pid, signal);
+        else
+            child.kill(signal);
+    }
+    catch {
+        // The process may have exited between the state check and the signal.
+    }
+}
+export const defaultGitRunner = (args, { cwd, auth, env: suppliedEnv, commitIdentityEnv, timeoutMs = DEFAULT_GIT_TIMEOUT_MS, signal }) => new Promise((resolve, reject) => {
+    if (!Number.isInteger(timeoutMs) || timeoutMs <= 0) {
+        reject(new Error('Git timeout must be a positive integer'));
+        return;
+    }
+    const secrets = auth ? [auth.token] : [];
+    assertSafeGitInvocation(args, secrets);
+    if (signal?.aborted) {
+        resolve({
+            exitCode: 130,
+            stdout: '',
+            stderr: 'Git operation cancelled',
+            cancelled: true,
+        });
+        return;
+    }
+    const baseEnv = auth
+        ? buildGitAuthEnvironment(auth, suppliedEnv)
+        : buildUnauthenticatedGitEnvironment(suppliedEnv);
+    let env = baseEnv;
+    if (commitIdentityEnv) {
+        assertCommitIdentityEnv(commitIdentityEnv);
+        env = { ...baseEnv, ...commitIdentityEnv };
+    }
+    const spawnArgs = auth ? ['-c', 'core.hooksPath=/dev/null', ...args] : args;
+    const child = spawn('git', spawnArgs, {
+        cwd,
+        env,
+        detached: process.platform !== 'win32',
+        stdio: auth ? ['ignore', 'pipe', 'pipe', 'pipe'] : ['ignore', 'pipe', 'pipe'],
+    });
+    let stdout = '';
+    let stderr = '';
+    let termination = null;
+    let settled = false;
+    let killTimer = null;
+    let hardStopTimer = null;
+    let timeoutTimer = null;
+    const cleanup = () => {
+        if (timeoutTimer)
+            clearTimeout(timeoutTimer);
+        if (killTimer)
+            clearTimeout(killTimer);
+        if (hardStopTimer)
+            clearTimeout(hardStopTimer);
+        signal?.removeEventListener('abort', cancel);
+    };
+    const finish = (exitCode) => {
+        if (settled)
+            return;
+        settled = true;
+        cleanup();
+        resolve({
+            exitCode,
+            stdout: redactSecrets(stdout, secrets),
+            stderr: redactSecrets(stderr, secrets),
+            timedOut: termination === 'timeout' || undefined,
+            cancelled: termination === 'cancelled' || undefined,
+        });
+    };
+    const terminate = (reason) => {
+        if (termination || settled)
+            return;
+        termination = reason;
+        signalGitProcess(child, 'SIGTERM');
+        killTimer = setTimeout(() => signalGitProcess(child, 'SIGKILL'), GIT_TERMINATION_GRACE_MS);
+        killTimer.unref?.();
+        hardStopTimer = setTimeout(() => finish(reason === 'timeout' ? 124 : 130), GIT_TERMINATION_GRACE_MS * 2);
+        hardStopTimer.unref?.();
+    };
+    const fail = (error) => {
+        if (settled)
+            return;
+        settled = true;
+        cleanup();
+        signalGitProcess(child, 'SIGKILL');
+        reject(error);
+    };
+    const cancel = () => terminate('cancelled');
+    timeoutTimer = setTimeout(() => terminate('timeout'), timeoutMs);
+    timeoutTimer.unref?.();
+    signal?.addEventListener('abort', cancel, { once: true });
+    if (signal?.aborted)
+        cancel();
+    const childStdout = child.stdout;
+    const childStderr = child.stderr;
+    if (!childStdout || !childStderr) {
+        fail(new Error('Could not capture Git process output'));
+        return;
+    }
+    childStdout.setEncoding('utf8');
+    childStderr.setEncoding('utf8');
+    childStdout.on('data', (chunk) => {
+        stdout += chunk;
+    });
+    childStderr.on('data', (chunk) => {
+        stderr += chunk;
+    });
+    child.on('error', (error) => {
+        if (termination) {
+            finish(termination === 'timeout' ? 124 : 130);
+            return;
+        }
+        fail(error);
+    });
+    child.on('close', (code) => {
+        finish(termination === 'timeout' ? 124 : termination === 'cancelled' ? 130 : (code ?? 1));
+    });
+    if (auth) {
+        const credentialPipe = child.stdio[3];
+        if (!credentialPipe) {
+            fail(new Error('Could not establish one-use Git credential pipe'));
+            return;
+        }
+        credentialPipe.on('error', () => {
+            /* EPIPE if Git fails before auth. */
+        });
+        if (termination) {
+            credentialPipe.destroy();
+            return;
+        }
+        try {
+            credentialPipe.end(`${auth.token}\n`);
+        }
+        catch (error) {
+            fail(error instanceof Error ? error : new Error('Could not write the one-use Git credential'));
+        }
+    }
+});
+export async function gitOk(runGit, args, options, operation) {
+    const result = await runGit(args, options);
+    if (result.timedOut)
+        throw new Error(`${operation} timed out`);
+    if (result.cancelled)
+        throw new Error(`${operation} cancelled`);
+    if (result.exitCode !== 0) {
+        const secrets = options.auth ? [options.auth.token] : [];
+        const detail = redactSecrets((result.stderr || result.stdout).trim(), secrets);
+        throw new Error(detail ? `${operation} failed: ${detail}` : `${operation} failed`);
+    }
+    return result.stdout.trim();
+}
+export async function revParse(runGit, cwd, ref) {
+    return gitOk(runGit, ['rev-parse', '--verify', ref], { cwd }, `resolve ${ref}`);
+}

package/dist/cloudflare-artifacts-git-remote.d.ts

@@ -0,0 +1,5 @@
+export declare const CLOUDFLARE_ARTIFACTS_HOST_PATTERN: RegExp;
+export declare const CLOUDFLARE_ARTIFACTS_GIT_PATH_PATTERN: RegExp;
+export declare function parseCanonicalCredentialFreeHttpsUrl(remote: unknown): URL | null;
+export declare function isValidCloudflareArtifactsGitRemote(remote: unknown): remote is string;
+//# sourceMappingURL=cloudflare-artifacts-git-remote.d.ts.map

package/dist/cloudflare-artifacts-git-remote.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cloudflare-artifacts-git-remote.d.ts","sourceRoot":"","sources":["../src/cloudflare-artifacts-git-remote.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,iCAAiC,QACC,CAAA;AAE/C,eAAO,MAAM,qCAAqC,QACmC,CAAA;AAErF,wBAAgB,oCAAoC,CAAC,MAAM,EAAE,OAAO,GAAG,GAAG,GAAG,IAAI,CAWhF;AAED,wBAAgB,mCAAmC,CAAC,MAAM,EAAE,OAAO,GAAG,MAAM,IAAI,MAAM,CAIrF"}

package/dist/cloudflare-artifacts-git-remote.js

@@ -0,0 +1,21 @@
+export const CLOUDFLARE_ARTIFACTS_HOST_PATTERN = /^[0-9a-f]{32}\.artifacts\.cloudflare\.net$/u;
+export const CLOUDFLARE_ARTIFACTS_GIT_PATH_PATTERN = /^\/git\/[A-Za-z0-9][A-Za-z0-9._-]{0,255}\/[A-Za-z0-9][A-Za-z0-9._-]{0,255}\.git$/u;
+export function parseCanonicalCredentialFreeHttpsUrl(remote) {
+    if (typeof remote !== 'string')
+        return null;
+    try {
+        const parsed = new URL(remote);
+        return parsed.protocol === 'https:' && !parsed.username && !parsed.password &&
+            !parsed.port && !parsed.search && !parsed.hash && parsed.toString() === remote
+            ? parsed
+            : null;
+    }
+    catch {
+        return null;
+    }
+}
+export function isValidCloudflareArtifactsGitRemote(remote) {
+    const parsed = parseCanonicalCredentialFreeHttpsUrl(remote);
+    return parsed !== null && CLOUDFLARE_ARTIFACTS_HOST_PATTERN.test(parsed.hostname) &&
+        CLOUDFLARE_ARTIFACTS_GIT_PATH_PATTERN.test(parsed.pathname);
+}

package/dist/dataset-transfer-download.d.ts

@@ -0,0 +1,2 @@
+export declare function transferDatasetDownload(datasetId: string, format: string, filename: string): Promise<void>;
+//# sourceMappingURL=dataset-transfer-download.d.ts.map

package/dist/dataset-transfer-download.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"dataset-transfer-download.d.ts","sourceRoot":"","sources":["../src/dataset-transfer-download.ts"],"names":[],"mappings":"AAmBA,wBAAsB,uBAAuB,CAAC,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAyChH"}

package/dist/dataset-transfer-download.js

@@ -0,0 +1,66 @@
+import { Effect, Either, Schedule } from 'effect';
+import { writeFileSync } from 'node:fs';
+import { datasetTransferTimeoutMs } from './dataset-transfer-policy.js';
+import { authedFetch } from './http.js';
+class DownloadHttpError extends Error {
+    status;
+    _tag = 'DownloadHttpError';
+    constructor(status, detail) {
+        super(`Download failed (HTTP ${status}): ${detail}`);
+        this.status = status;
+    }
+}
+class DownloadTransportError extends Error {
+    _tag = 'DownloadTransportError';
+    constructor(cause) {
+        super('Dataset download transport failed', { cause });
+    }
+}
+export async function transferDatasetDownload(datasetId, format, filename) {
+    const timeoutMs = datasetTransferTimeoutMs();
+    const download = Effect.tryPromise({
+        try: async (signal) => {
+            const response = await authedFetch(`/api/cli/datasets/${encodeURIComponent(datasetId)}/download?format=${encodeURIComponent(format)}`, { signal });
+            if (!response.ok)
+                throw new DownloadHttpError(response.status, await response.text());
+            return new Uint8Array(await response.arrayBuffer());
+        },
+        catch: cause => {
+            if (cause instanceof TypeError)
+                return new DownloadTransportError(cause);
+            return cause instanceof Error ? cause : new Error('Dataset download request failed', { cause });
+        },
+    }).pipe(Effect.timeoutFail({
+        duration: timeoutMs,
+        onTimeout: () => new Error('DATASET_TRANSFER_TIMEOUT: Dataset download exceeded its deadline'),
+    }), Effect.retry({
+        times: 2,
+        schedule: Schedule.spaced(100),
+        while: error => error instanceof DownloadTransportError || (error instanceof DownloadHttpError && [429, 502, 503, 504].includes(error.status)),
+    }));
+    const fileError = (cause) => new Error(`DATASET_DOWNLOAD_FILE: Could not save dataset to ${filename}: ${cause instanceof Error ? cause.message : String(cause)}`, { cause });
+    const write = (bytes) => Effect.try({
+        // Wait for the complete response, then retain the CLI's existing in-place
+        // write semantics, including inode metadata, hardlinks and symlink targets.
+        try: () => writeFileSync(filename, bytes, { mode: 0o600 }),
+        catch: fileError,
+    });
+    const controller = new AbortController();
+    const handleInterrupt = () => controller.abort();
+    process.once('SIGINT', handleInterrupt);
+    process.once('SIGTERM', handleInterrupt);
+    try {
+        const result = await Effect.runPromise(Effect.either(Effect.flatMap(download, write)), { signal: controller.signal });
+        if (Either.isLeft(result))
+            throw result.left;
+    }
+    catch (error) {
+        if (controller.signal.aborted)
+            throw new Error('DATASET_TRANSFER_INTERRUPTED: Dataset download cancelled');
+        throw error;
+    }
+    finally {
+        process.removeListener('SIGINT', handleInterrupt);
+        process.removeListener('SIGTERM', handleInterrupt);
+    }
+}

package/dist/dataset-transfer-policy.d.ts

@@ -0,0 +1,2 @@
+export declare function datasetTransferTimeoutMs(): number;
+//# sourceMappingURL=dataset-transfer-policy.d.ts.map

package/dist/dataset-transfer-policy.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"dataset-transfer-policy.d.ts","sourceRoot":"","sources":["../src/dataset-transfer-policy.ts"],"names":[],"mappings":"AAAA,wBAAgB,wBAAwB,IAAI,MAAM,CAOjD"}

package/dist/dataset-transfer-policy.js

@@ -0,0 +1,8 @@
+export function datasetTransferTimeoutMs() {
+    const configured = process.env.ORIZU_DATASET_TRANSFER_TIMEOUT_MS;
+    const timeoutMs = configured === undefined ? 120_000 : Number(configured);
+    if (!Number.isSafeInteger(timeoutMs) || timeoutMs < 1 || timeoutMs > 300_000) {
+        throw new Error('DATASET_TRANSFER_CONFIG: ORIZU_DATASET_TRANSFER_TIMEOUT_MS must be an integer from 1 to 300000');
+    }
+    return timeoutMs;
+}

package/dist/dataset-transfer-upload.d.ts

@@ -0,0 +1,42 @@
+export interface DatasetUploadResponse {
+    dataset: {
+        id: string;
+        name: string;
+        rowCount: number;
+        sourceType: string;
+        url?: string;
+    };
+    readmeVersion?: {
+        id: string;
+        version_num: number;
+        created_at: string;
+    } | null;
+}
+interface DatasetSnapshot {
+    datasetVersion: {
+        id: string;
+    };
+}
+type UploadRows = Array<Record<string, unknown>>;
+type UploadSource = {
+    kind: 'jsonl';
+    file: string;
+} | {
+    kind: 'rows';
+    rows: UploadRows;
+    sourceType: 'csv' | 'json' | 'jsonl';
+};
+export interface DatasetUploadOptions {
+    project: string;
+    name: string;
+    source: UploadSource;
+    readmeMarkdown: string | null;
+    json: boolean;
+    onChunk: (index: number, rowCount: number) => void;
+}
+export declare function transferDatasetUpload(options: DatasetUploadOptions): Promise<{
+    data: DatasetUploadResponse;
+    version: DatasetSnapshot;
+}>;
+export {};
+//# sourceMappingURL=dataset-transfer-upload.d.ts.map

package/dist/dataset-transfer-upload.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"dataset-transfer-upload.d.ts","sourceRoot":"","sources":["../src/dataset-transfer-upload.ts"],"names":[],"mappings":"AASA,MAAM,WAAW,qBAAqB;IACpC,OAAO,EAAE;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAC;QAAC,GAAG,CAAC,EAAE,MAAM,CAAA;KAAE,CAAA;IACzF,aAAa,CAAC,EAAE;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,WAAW,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAA;CAC/E;AAED,UAAU,eAAe;IAAG,cAAc,EAAE;QAAE,EAAE,EAAE,MAAM,CAAA;KAAE,CAAA;CAAE;AAE5D,KAAK,UAAU,GAAG,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAA;AAChD,KAAK,YAAY,GAAG;IAAE,IAAI,EAAE,OAAO,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,GAAG;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,UAAU,CAAC;IAAC,UAAU,EAAE,KAAK,GAAG,MAAM,GAAG,OAAO,CAAA;CAAE,CAAA;AAE9H,MAAM,WAAW,oBAAoB;IACnC,OAAO,EAAE,MAAM,CAAA;IACf,IAAI,EAAE,MAAM,CAAA;IACZ,MAAM,EAAE,YAAY,CAAA;IACpB,cAAc,EAAE,MAAM,GAAG,IAAI,CAAA;IAC7B,IAAI,EAAE,OAAO,CAAA;IACb,OAAO,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,KAAK,IAAI,CAAA;CACnD;AAoHD,wBAAgB,qBAAqB,CAAC,OAAO,EAAE,oBAAoB,GAAG,OAAO,CAAC;IAAE,IAAI,EAAE,qBAAqB,CAAC;IAAC,OAAO,EAAE,eAAe,CAAA;CAAE,CAAC,CAEvI"}

package/dist/dataset-transfer-upload.js

@@ -0,0 +1,110 @@
+import { Context, Effect, Either, Layer } from 'effect';
+import { ensureDatasetUploadSnapshot } from './dataset-upload-snapshot.js';
+import { datasetTransferTimeoutMs } from './dataset-transfer-policy.js';
+import { extractErrorMessage } from './error-response.js';
+import { authedFetch } from './http.js';
+import { parseJsonResponse, sanitizeTerminalText } from './json-response.js';
+import { streamJsonlRowChunks } from './jsonl-stream.js';
+class DatasetWriteFailure extends Error {
+    outcome;
+    _tag = 'DatasetWriteFailure';
+    constructor(message, outcome, options) {
+        super(message, options);
+        this.outcome = outcome;
+    }
+}
+class DatasetInputFailure extends Error {
+    _tag = 'DatasetInputFailure';
+    constructor(cause) { super(cause instanceof Error ? cause.message : String(cause), { cause }); }
+}
+const DatasetUploadApi = Context.GenericTag('orizu/DatasetUploadApi');
+// Only the existing authentication client's rejected-token refresh may replay a
+// request. No transient/transport retries belong around these non-idempotent writes.
+function post(path, body, context, prefix, timeoutMs) {
+    return Effect.suspend(() => {
+        let requestStarted = false;
+        return Effect.tryPromise({
+            try: async (signal) => {
+                const response = await authedFetch(path, {
+                    method: 'POST', headers: { 'Content-Type': 'application/json' },
+                    body: JSON.stringify(body), signal,
+                }, () => { requestStarted = true; });
+                if (!response.ok) {
+                    throw new DatasetWriteFailure(`${prefix}: ${await extractErrorMessage(response)}`, response.status >= 500 ? 'unknown' : 'rejected');
+                }
+                return await parseJsonResponse(response, context);
+            },
+            catch: cause => cause instanceof DatasetWriteFailure ? cause : new DatasetWriteFailure(cause instanceof Error ? cause.message : String(cause), requestStarted ? 'unknown' : 'rejected', { cause }),
+        }).pipe(Effect.timeoutFail({
+            duration: timeoutMs,
+            onTimeout: () => new DatasetWriteFailure(`DATASET_TRANSFER_TIMEOUT: ${context} exceeded its deadline`, requestStarted ? 'unknown' : 'rejected'),
+        }));
+    });
+}
+function liveUploadApi(options) {
+    const timeoutMs = datasetTransferTimeoutMs();
+    return Layer.succeed(DatasetUploadApi, {
+        create: (rows, sourceType) => post('/api/cli/datasets/upload', {
+            projectSlug: options.project, name: options.name, rows, sourceType,
+            ...(options.readmeMarkdown !== null ? { readmeMarkdown: options.readmeMarkdown } : {}),
+        }, 'Dataset upload', 'Upload failed', timeoutMs),
+        append: (id, rows) => post(`/api/cli/datasets/${encodeURIComponent(id)}/rows`, { rows }, 'Dataset append', 'Append failed', timeoutMs),
+        snapshot: id => post(`/api/cli/datasets/${encodeURIComponent(id)}/versions`, {}, 'Dataset version create', 'Failed to create dataset version', timeoutMs),
+    });
+}
+async function expectedPromise(program) {
+    const result = await Effect.runPromise(Effect.either(program));
+    if (Either.isLeft(result))
+        throw result.left;
+    return result.right;
+}
+function uploadProgram(options) {
+    return Effect.gen(function* () {
+        const api = yield* DatasetUploadApi;
+        const chunks = yield* Effect.acquireRelease(Effect.sync(() => options.source.kind === 'jsonl'
+            ? streamJsonlRowChunks(options.source.file)
+            : (async function* () { if (options.source.kind === 'rows')
+                yield options.source.rows; })()), iterator => Effect.promise(async () => { await iterator.return(undefined); }));
+        let data;
+        let confirmedRows = 0;
+        let chunkIndex = 0;
+        while (true) {
+            const next = yield* Effect.tryPromise({ try: () => chunks.next(), catch: cause => new DatasetInputFailure(cause) }).pipe(Effect.mapError(error => new Error(data
+                ? `Upload stopped while reading the next JSONL chunk: ${error.message}\nDataset ${sanitizeTerminalText(data.dataset.name)} (${sanitizeTerminalText(data.dataset.id)}) was created and ${confirmedRows} rows were uploaded. Fix the file, remove the first ${confirmedRows} rows, and run orizu datasets append --dataset ${data.dataset.id} --file <remaining-file>.`
+                : error.message)));
+            if (next.done)
+                break;
+            chunkIndex++;
+            if (options.source.kind === 'jsonl')
+                yield* Effect.sync(() => options.onChunk(chunkIndex, next.value.length));
+            const write = data
+                ? api.append(data.dataset.id, next.value).pipe(Effect.map(result => ({
+                    data: { ...data, dataset: { ...data.dataset, rowCount: result.dataset.rowCount } },
+                    added: result.appendedCount,
+                })))
+                : api.create(next.value, options.source.kind === 'jsonl' ? 'jsonl' : options.source.sourceType).pipe(Effect.map(result => ({ data: result, added: result.dataset.rowCount })));
+            const result = yield* write.pipe(Effect.mapError(error => {
+                if (error.outcome === 'unknown') {
+                    return new Error(`DATASET_UPLOAD_UNCERTAIN: Chunk ${chunkIndex} may have committed, but its outcome could not be confirmed: ${error.message}\n` +
+                        (data ? `Dataset ${sanitizeTerminalText(data.dataset.id)} has ${confirmedRows} confirmed uploaded rows. ` : 'Dataset creation could not be confirmed. ') +
+                        'Inspect the dataset and reconcile row IDs before retrying; do not replay this chunk blindly.');
+                }
+                return new Error(data
+                    ? `Chunk ${chunkIndex} failed: ${error.message}\nDataset ${sanitizeTerminalText(data.dataset.name)} (${sanitizeTerminalText(data.dataset.id)}) was created and ${confirmedRows} rows were uploaded. To retry, remove the first ${confirmedRows} rows from your file and run orizu datasets append --dataset ${data.dataset.id} --file <remaining-file>.`
+                    : error.message);
+            }));
+            data = result.data;
+            confirmedRows += result.added;
+        }
+        if (!data)
+            return yield* Effect.fail(new Error('Dataset file contains no rows'));
+        const version = yield* Effect.tryPromise({
+            try: () => ensureDatasetUploadSnapshot(data.dataset, options.json, id => expectedPromise(api.snapshot(id))),
+            catch: cause => cause instanceof Error ? cause : new Error(String(cause)),
+        });
+        return { data, version };
+    });
+}
+export function transferDatasetUpload(options) {
+    return expectedPromise(uploadProgram(options).pipe(Effect.scoped, Effect.provide(liveUploadApi(options))));
+}