@chalksurf/cli 0.1.0 → 0.2.0

package/dist/lib/source-resolver.js

@@ -1,5 +1,5 @@
 import { createWriteStream } from 'node:fs';
-import { mkdtemp, readdir, rm, stat } from 'node:fs/promises';
+import { mkdtemp, open, readdir, rm, stat } from 'node:fs/promises';
 import { tmpdir } from 'node:os';
 import { basename, isAbsolute, relative, resolve } from 'node:path';
 import { Readable } from 'node:stream';
@@ -8,6 +8,8 @@ import pLimit from 'p-limit';
 import { CliCommandError } from './cli-error.js';
 const sourceResolutionExitCode = 4;
 const defaultMaxConcurrentUrlDownloads = 4;
+const htmlSniffBytes = 1024;
+const genericMimeTypes = new Set(['application/octet-stream', 'binary/octet-stream']);
 const createSourceResolutionError = (message) => {
     return new CliCommandError(message, sourceResolutionExitCode);
 };
@@ -85,7 +87,44 @@ const deriveRelativePathFromUrl = (url) => {
     }
     return fileName;
 };
-const resolveLocalFileSource = async ({ source, cwd, }) => {
+const normalizeMimeType = (contentTypeHeaderValue) => {
+    const mimeType = contentTypeHeaderValue?.split(';', 1)[0]?.trim().toLowerCase();
+    return mimeType || undefined;
+};
+const stripLeadingHtmlComments = (value) => {
+    let normalizedValue = value.trimStart();
+    while (normalizedValue.startsWith('<!--')) {
+        const commentEndIndex = normalizedValue.indexOf('-->');
+        if (commentEndIndex === -1) {
+            return normalizedValue;
+        }
+        normalizedValue = normalizedValue.slice(commentEndIndex + 3).trimStart();
+    }
+    return normalizedValue;
+};
+const looksLikeHtmlDocument = (value) => {
+    const normalizedValue = stripLeadingHtmlComments(value.replace(/^\uFEFF/, ''));
+    return /^<(?:!doctype html|html|head|body)\b/i.test(normalizedValue);
+};
+const sniffMimeType = async ({ filePath, responseMimeType }) => {
+    if (responseMimeType && !genericMimeTypes.has(responseMimeType)) {
+        return responseMimeType;
+    }
+    const fileHandle = await open(filePath, 'r');
+    try {
+        const buffer = Buffer.alloc(htmlSniffBytes);
+        const { bytesRead } = await fileHandle.read(buffer, 0, buffer.length, 0);
+        const filePrefix = buffer.subarray(0, bytesRead).toString('utf8');
+        if (looksLikeHtmlDocument(filePrefix)) {
+            return 'text/html';
+        }
+        return responseMimeType;
+    }
+    finally {
+        await fileHandle.close();
+    }
+};
+const resolveLocalFileSource = async ({ source, cwd, sourceInputIndex, }) => {
     const resolvedPath = resolve(cwd, source.path);
     await ensureExistingFile(resolvedPath, 'Local source');
     const relativePath = normalizeRelativePath(source.relativePath ?? basename(resolvedPath));
@@ -96,10 +135,12 @@ const resolveLocalFileSource = async ({ source, cwd, }) => {
             filePath: resolvedPath,
             fileName: basename(relativePath),
             relativePath,
+            sourceInputIndex,
+            sourceInput: source,
         },
     ];
 };
-const resolveDirectorySource = async ({ source, cwd, }) => {
+const resolveDirectorySource = async ({ source, cwd, sourceInputIndex, }) => {
     const resolvedDirectoryPath = resolve(cwd, source.path);
     const resolvedRelativeRoot = resolve(cwd, source.relativeRoot ?? source.path);
     await ensureExistingDirectory(resolvedDirectoryPath, 'Directory source');
@@ -119,10 +160,12 @@ const resolveDirectorySource = async ({ source, cwd, }) => {
             filePath,
             fileName: basename(relativePath),
             relativePath,
+            sourceInputIndex,
+            sourceInput: source,
         };
     });
 };
-const resolveUrlSource = async ({ source, fetchImpl, }) => {
+const resolveUrlSource = async ({ source, sourceInputIndex, }) => {
     let parsedUrl;
     try {
         parsedUrl = new URL(source.url);
@@ -137,22 +180,30 @@ const resolveUrlSource = async ({ source, fetchImpl, }) => {
     const downloadDirectoryPath = await mkdtemp(resolve(tmpdir(), 'chalksurf-cli-source-'));
     const downloadedFilePath = resolve(downloadDirectoryPath, basename(relativePath));
     try {
-        const response = await fetchImpl(source.url);
+        const response = await fetch(source.url);
         if (!response.ok) {
             throw createSourceResolutionError(`Failed to download "${source.url}": HTTP ${response.status}${response.statusText ? ` ${response.statusText}` : ''}.`);
         }
         if (!response.body) {
             throw createSourceResolutionError(`Failed to download "${source.url}": response body was empty.`);
         }
+        const responseMimeType = normalizeMimeType(response.headers.get('content-type'));
         await pipeline(Readable.fromWeb(response.body), createWriteStream(downloadedFilePath));
+        const mimeType = await sniffMimeType({
+            filePath: downloadedFilePath,
+            responseMimeType,
+        });
         return [
             {
                 kind: 'url',
                 input: source.url,
                 filePath: downloadedFilePath,
                 fileName: basename(relativePath),
+                mimeType,
                 relativePath,
+                sourceInputIndex,
                 cleanupPath: downloadDirectoryPath,
+                sourceInput: source,
             },
         ];
     }
@@ -182,16 +233,27 @@ export const cleanupResolvedSources = async (resolvedSources) => {
         await rm(cleanupPath, { force: true, recursive: true });
     }));
 };
-export const resolveSources = async ({ sources, cwd = process.cwd(), fetchImpl = fetch, maxConcurrentUrlDownloads = defaultMaxConcurrentUrlDownloads, }) => {
+export const resolveSources = async ({ sources, cwd = process.cwd(), maxConcurrentUrlDownloads = defaultMaxConcurrentUrlDownloads, }) => {
     const limitUrlDownloads = pLimit(Math.max(1, maxConcurrentUrlDownloads));
-    const settledSourceGroups = await Promise.allSettled(sources.map(async (source) => {
+    const settledSourceGroups = await Promise.allSettled(sources.map(async (source, sourceInputIndex) => {
         if (source.kind === 'local') {
-            return await resolveLocalFileSource({ source, cwd });
+            return await resolveLocalFileSource({
+                source: source,
+                cwd,
+                sourceInputIndex,
+            });
         }
         if (source.kind === 'directory') {
-            return await resolveDirectorySource({ source, cwd });
+            return await resolveDirectorySource({
+                source: source,
+                cwd,
+                sourceInputIndex,
+            });
         }
-        return await limitUrlDownloads(() => resolveUrlSource({ source, fetchImpl }));
+        return await limitUrlDownloads(() => resolveUrlSource({
+            source: source,
+            sourceInputIndex,
+        }));
     }));
     const resolvedSources = settledSourceGroups.flatMap((result) => (result.status === 'fulfilled' ? result.value : []));
     const firstRejectedResult = settledSourceGroups.find((result) => result.status === 'rejected');

package/dist/lib/translation-languages.js

@@ -0,0 +1 @@
+export const translationLanguages = ['english', 'hungarian', 'german', 'french', 'spanish', 'italian'];

package/dist/lib/user-jobs.js

@@ -1,4 +1,4 @@
-import { CliCommandError } from './cli-error.js';
+import { CliCommandError, createSerializableCliError } from './cli-error.js';
 import { mapApiErrorToCliError } from './session.js';
 const isTerminalJob = (job) => {
     return ['completed', 'failed'].includes(job.status);
@@ -41,6 +41,21 @@ export const getWaitExitCode = ({ jobs, timedOut }) => {
     }
     return 0;
 };
+export const getWaitError = ({ jobs, timedOut }) => {
+    if (timedOut) {
+        return createSerializableCliError({
+            exitCode: 6,
+            message: 'Timed out before all jobs completed.',
+        });
+    }
+    if (jobs.some((job) => job.status === 'failed')) {
+        return createSerializableCliError({
+            exitCode: 7,
+            message: 'One or more jobs failed.',
+        });
+    }
+    return undefined;
+};
 export const waitForCliJobs = async ({ getUserJob, jobIds, maxPollIntervalMs = 5000, now, sleep, timeoutMs = 300000, }) => {
     const startedAt = now();
     let nextPollDelayMs = 1000;

package/docs/agents.md

@@ -0,0 +1,195 @@
+# Agent And Codex Guide
+
+Use this flow when ChalkSurf is driven by Codex, CI, or another orchestration layer.
+
+## Authentication Strategy
+
+Prefer environment variables over interactive config:
+
+```bash
+export CHALKSURF_BASE_URL=https://api.chalksurf.com
+export CHALKSURF_TOKEN=cs_cli_...
+export CHALKSURF_ORGANIZATION_ID=org_123
+```
+
+Check the session in machine-readable mode:
+
+```bash
+chalksurf auth status --json
+```
+
+Avoid `auth login` for short-lived runs unless you explicitly want local persistence.
+
+## Command Contract
+
+For agent-driven imports, always use:
+
+- `--manifest -` or a generated manifest file
+- `--wait` when the next step depends on completed imports
+- `--json` so the response stays machine-readable
+
+Recommended invocation shape:
+
+```bash
+chalksurf sheet import --manifest - --wait --json
+```
+
+In `--json` mode:
+
+- stdout contains a stable JSON envelope
+- known failures still produce JSON on stdout
+- stderr is reserved for unexpected runtime failures
+
+Wait-style failures still include a populated `result` payload, so agents can inspect partial outcomes on exit code `6` or `7`.
+
+## Manifest Design
+
+Prefer manifests over large positional argument lists.
+
+Why:
+
+- multi-source imports stay explicit
+- each logical source can carry a stable `sourceId`
+- sheet imports can group multiple source files into one resulting sheet
+- sheet imports carry `targetFolderPath`, `title`, and `translateTo` at the sheet level
+- agents can correlate import results back to the discovered source set
+
+Use `sourceId` whenever a browsing step or upstream scraper already has a stable identifier:
+
+```json
+{
+  "organizationId": "org_123",
+  "wait": true,
+  "sheets": [
+    {
+      "targetFolderPath": "OKTV/2014",
+      "title": "OKTV 2014 Round 1",
+      "translateTo": ["english"],
+      "sources": [
+        {
+          "sourceId": "oktv-2014-round-1-part-a",
+          "kind": "url",
+          "url": "https://example.com/oktv-2014-round-1-part-a.pdf"
+        },
+        {
+          "sourceId": "oktv-2014-round-1-part-b",
+          "kind": "url",
+          "url": "https://example.com/oktv-2014-round-1-part-b.pdf"
+        }
+      ]
+    }
+  ]
+}
+```
+
+For sheet imports, `targetFolderPath` is the destination of the resulting sheet. Use `null` for the root folder. Source-level `relativePath` is only a filename or source-label override and should not be used as a destination path.
+
+Further reference:
+
+- [Manifest reference](./manifest.md)
+- [Sheet import schema](../schemas/sheet-import-manifest.schema.json)
+- [Exercise import schema](../schemas/exercise-import-manifest.schema.json)
+- [Exercise solution import schema](../schemas/exercise-solution-import-manifest.schema.json)
+
+## JSON Result Shape
+
+Import commands normalize results around:
+
+- `request`
+- `sources`
+- `jobs`
+- `summary`
+
+Each resolved source includes stable correlation fields:
+
+- `sourceIndex`
+- `sourceInputIndex`
+- `sourceId`
+
+Each job includes:
+
+- `jobId`
+- `sourceIndexes`
+- `sourceIds`
+- `status`
+
+`summary` includes:
+
+- source and job counts
+- per-status counts
+- `timedOut`
+
+## Recommended Control Flow
+
+1. Discover candidate URLs or local files.
+2. Filter them to the target scope.
+3. Build a manifest with stable `sourceId` values.
+4. Pipe the manifest into the CLI with `--wait --json`.
+5. Inspect `ok`, `result.summary`, and `result.jobs`.
+6. Retry only the failed or timed-out source set.
+
+For example:
+
+```bash
+cat import.json | chalksurf sheet import --manifest - --wait --json
+```
+
+On success, the envelope looks like:
+
+```json
+{
+  "schemaVersion": "v1",
+  "command": "sheet import",
+  "ok": true,
+  "result": {
+    "request": {},
+    "sources": [],
+    "jobs": [],
+    "summary": {
+      "timedOut": false
+    }
+  },
+  "warnings": []
+}
+```
+
+On timeout or job failure, `ok` becomes `false`, `error.code` is stable, and `result` still contains the normalized import data needed for retries.
+
+## Codex Workflow Example
+
+Target task:
+
+Import all OKTV exercise sheets between 2010 and 2020 from an archive page, and translate the non-English sheets to English.
+
+Recommended workflow:
+
+1. Codex browses the archive page and collects the relevant links.
+2. Codex filters the links to years `2010` through `2020`.
+3. Codex builds a sheet-import manifest with one `sheets[]` entry per resulting sheet.
+4. Codex groups multiple source files inside one `sources[]` array when they belong to the same resulting sheet.
+5. Codex sets `translateTo: ["english"]` on the sheet entries that should produce an English translation.
+6. Codex runs `chalksurf sheet import --manifest - --wait --json`.
+7. Codex inspects `result.summary.failed`, `result.summary.timedOut`, and `jobs` to decide whether to retry or report failures.
+
+Minimal shell shape:
+
+```bash
+cat import.json | chalksurf sheet import --manifest - --wait --json
+```
+
+The discovery step belongs outside the ChalkSurf CLI. The CLI contract starts at a concrete manifest and ends at structured import results.
+
+## Failure Handling
+
+Use the exit code and JSON `error.code` together:
+
+- `2` / `usage_error`: the manifest or command input is invalid
+- `3` / `not_authenticated`: missing or invalid token
+- `4` / `source_resolution_failed`: local file, directory, or URL preparation failed
+- `5` / `api_error`: the ChalkSurf API rejected the request
+- `6` / `wait_timed_out`: jobs did not finish in time
+- `7` / `job_failed`: one or more jobs finished with failure
+
+Reference:
+
+- [Exit codes and JSON errors](./exit-codes.md)

package/docs/examples/exercise-import-manifest.json

@@ -0,0 +1,13 @@
+{
+  "organizationId": "org_123",
+  "exerciseSheetId": "00000000-0000-4000-8000-000000000111",
+  "wait": true,
+  "sources": [
+    {
+      "sourceId": "problem-1",
+      "kind": "local",
+      "path": "./imports/problem-1.pdf",
+      "relativePath": "Problem Set/problem-1.pdf"
+    }
+  ]
+}

package/docs/examples/exercise-solution-import-manifest.json

@@ -0,0 +1,13 @@
+{
+  "organizationId": "org_123",
+  "exerciseId": "00000000-0000-4000-8000-000000000222",
+  "wait": true,
+  "sources": [
+    {
+      "sourceId": "problem-1-solution",
+      "kind": "local",
+      "path": "./imports/problem-1-solution.pdf",
+      "relativePath": "Solutions/problem-1.pdf"
+    }
+  ]
+}

package/docs/examples/sheet-import-manifest.json

@@ -0,0 +1,33 @@
+{
+  "organizationId": "org_123",
+  "wait": true,
+  "sheets": [
+    {
+      "targetFolderPath": "OKTV/2014",
+      "title": "OKTV 2014 Round 1",
+      "translateTo": ["english"],
+      "sources": [
+        {
+          "sourceId": "oktv-2014-round-1-part-a",
+          "kind": "url",
+          "url": "https://example.com/oktv-2014-round-1-part-a.pdf"
+        },
+        {
+          "sourceId": "oktv-2014-round-1-part-b",
+          "kind": "url",
+          "url": "https://example.com/oktv-2014-round-1-part-b.pdf"
+        }
+      ]
+    },
+    {
+      "targetFolderPath": null,
+      "sources": [
+        {
+          "sourceId": "practice-sheet",
+          "kind": "local",
+          "path": "./practice-sheet.pdf"
+        }
+      ]
+    }
+  ]
+}

package/docs/exit-codes.md

@@ -0,0 +1,57 @@
+# Exit Codes And JSON Errors
+
+The CLI uses stable exit codes so automation can branch without parsing human text.
+
+## Exit Codes
+
+| Exit code | Error code | Meaning |
+| --- | --- | --- |
+| `0` | n/a | Success |
+| `1` | `unexpected_error` | Unhandled runtime failure |
+| `2` | `usage_error` | Invalid arguments, invalid manifest, or missing required config such as `--base-url` |
+| `3` | `not_authenticated` | Missing token or rejected CLI token |
+| `4` | `source_resolution_failed` | Local file, directory, URL, or relative path resolution failed |
+| `5` | `api_error` | The ChalkSurf API returned a non-auth failure |
+| `6` | `wait_timed_out` | Waiting ended before all jobs reached a terminal state |
+| `7` | `job_failed` | One or more jobs completed with failure |
+
+These same codes are mirrored in `error.exitCode` and `error.code` inside the JSON envelope.
+
+## JSON Envelope
+
+In `--json` mode, stdout contains a stable envelope:
+
+```json
+{
+  "schemaVersion": "v1",
+  "command": "sheet import",
+  "ok": true,
+  "result": {},
+  "warnings": []
+}
+```
+
+Known failures also return JSON on stdout:
+
+```json
+{
+  "schemaVersion": "v1",
+  "command": "sheet import",
+  "ok": false,
+  "result": {},
+  "error": {
+    "code": "source_resolution_failed",
+    "message": "Local source \"/tmp/imports/missing.pdf\" does not exist.",
+    "exitCode": 4,
+    "retryable": false
+  },
+  "warnings": []
+}
+```
+
+## Contract Notes
+
+- `warnings` contains informational messages that would otherwise be printed for humans.
+- In `--json` mode, stderr is reserved for unexpected runtime failures only.
+- Wait-style commands and imports keep their `result` payload even when `ok` is `false`.
+- Import commands normalize their `result` payloads around `request`, `sources`, `jobs`, and `summary`.

package/docs/manifest.md

@@ -0,0 +1,129 @@
+# Manifest Reference
+
+ChalkSurf uses JSON manifests for batch and agent-driven imports.
+
+One manifest file is passed to exactly one command:
+
+- `chalksurf sheet import --manifest <path|->`
+- `chalksurf exercise import --manifest <path|->`
+- `chalksurf exercise import-solution --manifest <path|->`
+
+Use `--manifest -` to pipe JSON on stdin.
+
+## Common Fields
+
+Every manifest is a JSON object with these shared metadata fields:
+
+| Field | Type | Required | Notes |
+| --- | --- | --- | --- |
+| `organizationId` | string | no | Overrides the default organization for this invocation. |
+| `wait` | boolean | no | Behaves like `--wait`. The CLI also accepts `--wait`, which wins if set. |
+
+Every source object supports these common fields. For `sheet import`, sources appear inside `sheets[].sources[]`. For the exercise import commands, they appear in top-level `sources[]`.
+
+| Field | Type | Required | Notes |
+| --- | --- | --- | --- |
+| `sourceId` | string | no | Optional stable identifier for correlating source inputs to results. Must be unique within the manifest. |
+| `kind` | `local` \| `directory` \| `url` | yes | Selects the source shape. |
+
+### Source Kinds
+
+`local`
+
+| Field | Type | Required | Notes |
+| --- | --- | --- | --- |
+| `path` | string | yes | Local file path relative to `cwd` unless already absolute. |
+| `relativePath` | string | no | Stored path shown to ChalkSurf. Defaults to the basename of `path`. |
+
+`directory`
+
+| Field | Type | Required | Notes |
+| --- | --- | --- | --- |
+| `path` | string | yes | Directory to expand recursively. |
+| `relativeRoot` | string | no | Base directory used to compute stored relative paths. Defaults to `path`. |
+
+`url`
+
+| Field | Type | Required | Notes |
+| --- | --- | --- | --- |
+| `url` | string | yes | HTTP(S) URL to download before import. |
+| `relativePath` | string | no | Stored path shown to ChalkSurf. Defaults to the filename derived from the URL. |
+
+## Command-Specific Fields
+
+### Sheet Import
+
+Top-level fields:
+
+| Field | Type | Required | Notes |
+| --- | --- | --- | --- |
+| `sheets` | array | yes | One or more sheets to import. Each sheet may contain one or more source files. |
+
+Sheet fields:
+
+| Field | Type | Required | Notes |
+| --- | --- | --- | --- |
+| `targetFolderPath` | string \| null | yes | Destination folder for the resulting sheet. Use `null` for the root folder. |
+| `sources` | array | yes | One or more source files that belong to this sheet, in import order. |
+| `title` | string | no | Overrides the imported sheet title. |
+| `translateTo` | string[] | no | Target translation languages. Must be unique and non-empty when present. |
+
+Sheet import metadata is sheet-level, not source-level. Do not put `title` or `translateTo` on individual sources inside `sheets[].sources[]`.
+
+Source-level `relativePath` is only a source label or filename override. It does not choose the destination folder for grouped sheet imports; `targetFolderPath` is the only destination field.
+
+Canonical example:
+
+- [docs/examples/sheet-import-manifest.json](./examples/sheet-import-manifest.json)
+- [schemas/sheet-import-manifest.schema.json](../schemas/sheet-import-manifest.schema.json)
+
+### Exercise Import
+
+Top-level fields:
+
+| Field | Type | Required | Notes |
+| --- | --- | --- | --- |
+| `sources` | array | yes | One or more import sources. |
+| `exerciseSheetId` | string | no | Default target sheet for the imported exercises. Can still be overridden by `--sheet-id`. |
+
+Per-source fields:
+
+- Only the common source fields are allowed.
+- `title` and `translateTo` are rejected for this command.
+
+Canonical example:
+
+- [docs/examples/exercise-import-manifest.json](./examples/exercise-import-manifest.json)
+- [schemas/exercise-import-manifest.schema.json](../schemas/exercise-import-manifest.schema.json)
+
+### Exercise Solution Import
+
+Top-level fields:
+
+| Field | Type | Required | Notes |
+| --- | --- | --- | --- |
+| `sources` | array | yes | One or more import sources. |
+| `exerciseId` | string | no | Default target exercise for the imported solution files. The command still requires an exercise id overall, either here or positionally. |
+
+Per-source fields:
+
+- Only the common source fields are allowed.
+- `title` and `translateTo` are rejected for this command.
+
+Canonical example:
+
+- [docs/examples/exercise-solution-import-manifest.json](./examples/exercise-solution-import-manifest.json)
+- [schemas/exercise-solution-import-manifest.schema.json](../schemas/exercise-solution-import-manifest.schema.json)
+
+## Validation Notes
+
+The CLI validates more than the JSON schema can express on its own:
+
+- `sourceId` values must be unique within one manifest.
+- Sheet import `translateTo` values must be unique within one sheet.
+- `relativePath` cannot be empty or contain `..`.
+- Sheet import `targetFolderPath` must be `null` or a normalized folder path without `.` or `..` segments.
+- `directory` imports must resolve to at least one file.
+- URL imports must use `http` or `https`.
+
+Treat the schema files as a reference format and the CLI as the final validator.