@raintree-technology/docpull-sdk 6.5.0

package/README.md

@@ -0,0 +1,63 @@
+# @raintree-technology/docpull-sdk
+
+TypeScript SDK for [docpull](https://github.com/raintree-technology/docpull).
+It reads local context packs (the v3 pack contract) and runs the `docpull`
+CLI from Node or Bun. Reading a pack never touches the network; fetching
+requires the `docpull` CLI on PATH.
+
+## Install
+
+```bash
+bun add @raintree-technology/docpull-sdk   # or: npm install @raintree-technology/docpull-sdk
+```
+
+## Read a pack
+
+```ts
+import { readCorpusManifest, readDocuments, readPack } from "@raintree-technology/docpull-sdk";
+
+const manifest = await readCorpusManifest("./packs/example");
+console.log(manifest.record_count, manifest.records[0]?.url);
+
+// Stream records one line at a time.
+for await (const record of readDocuments("./packs/example")) {
+  console.log(record.document_id, record.token_count);
+}
+
+// Or load everything at once.
+const pack = await readPack("./packs/example");
+console.log(pack.documents.length);
+```
+
+`readDocuments` prefers `documents.ndjson` and falls back to
+`documents.jsonl`. Types (`CorpusManifest`, `ManifestRecord`,
+`DocumentRecord`) mirror the field names written by the Python side and keep
+unknown fields as forward-compatible metadata.
+
+## Run the CLI
+
+```ts
+import { fetchToPack, runDocpull } from "@raintree-technology/docpull-sdk";
+
+// docpull https://example.com/docs -o ./packs/example --budget 0
+await fetchToPack("https://example.com/docs", "./packs/example", { budget: 0 });
+
+// Any other subcommand.
+const result = await runDocpull(["pack", "validate", "./packs/example"]);
+console.log(result.stdout);
+```
+
+Commands are spawned with an argument array and `shell: false`. A non-zero
+exit rejects with a `DocpullCliError` that carries `exitCode` and `stderr`.
+
+## Development
+
+```bash
+bun install        # from the repo root
+bun test           # from sdk/js
+bun run typecheck  # tsc --noEmit
+```
+
+## License
+
+MIT

package/dist/cli.d.ts

@@ -0,0 +1,53 @@
+/**
+ * Helpers for running the docpull CLI from JavaScript.
+ *
+ * Commands are spawned with an argument array and `shell: false`, so no
+ * shell interpolation happens. A non-zero exit rejects with the captured
+ * stderr in the error.
+ */
+export interface SpawnedProcessStream {
+    on(event: "data", listener: (chunk: unknown) => void): void;
+}
+/** Minimal child process surface used by `runDocpull`; injectable in tests. */
+export interface SpawnedProcess {
+    stdout: SpawnedProcessStream | null;
+    stderr: SpawnedProcessStream | null;
+    on(event: "error", listener: (error: Error) => void): void;
+    on(event: "close", listener: (code: number | null) => void): void;
+}
+export interface SpawnCallOptions {
+    cwd?: string;
+    env?: Record<string, string | undefined>;
+    shell: false;
+}
+export type SpawnFunction = (command: string, args: readonly string[], options: SpawnCallOptions) => SpawnedProcess;
+export interface RunDocpullOptions {
+    /** Executable to run. Defaults to `docpull` resolved from PATH. */
+    bin?: string;
+    cwd?: string;
+    env?: Record<string, string | undefined>;
+    /** Spawn implementation override, used by tests to avoid real processes. */
+    spawnFn?: SpawnFunction;
+}
+export interface RunDocpullResult {
+    command: string;
+    args: string[];
+    stdout: string;
+    stderr: string;
+    exitCode: number;
+}
+export declare class DocpullCliError extends Error {
+    readonly exitCode: number | null;
+    readonly stderr: string;
+    constructor(message: string, exitCode: number | null, stderr: string);
+}
+/** Run the docpull CLI with the given arguments and capture its output. */
+export declare function runDocpull(args: readonly string[], options?: RunDocpullOptions): Promise<RunDocpullResult>;
+export interface FetchToPackOptions extends RunDocpullOptions {
+    /** Value for `--budget`; pass 0 to disable the network budget. */
+    budget?: number;
+    /** Extra CLI arguments appended after the generated ones. */
+    extraArgs?: readonly string[];
+}
+/** Fetch a URL into a pack directory: `docpull <url> -o <outputDir>`. */
+export declare function fetchToPack(url: string, outputDir: string, options?: FetchToPackOptions): Promise<RunDocpullResult>;

package/dist/cli.js

@@ -0,0 +1,77 @@
+/**
+ * Helpers for running the docpull CLI from JavaScript.
+ *
+ * Commands are spawned with an argument array and `shell: false`, so no
+ * shell interpolation happens. A non-zero exit rejects with the captured
+ * stderr in the error.
+ */
+import { spawn } from "node:child_process";
+export class DocpullCliError extends Error {
+    exitCode;
+    stderr;
+    constructor(message, exitCode, stderr) {
+        super(message);
+        this.name = "DocpullCliError";
+        this.exitCode = exitCode;
+        this.stderr = stderr;
+    }
+}
+const defaultSpawn = (command, args, options) => {
+    const child = spawn(command, [...args], {
+        cwd: options.cwd,
+        env: options.env,
+        shell: false,
+    });
+    return {
+        stdout: child.stdout,
+        stderr: child.stderr,
+        on(event, listener) {
+            child.on(event, listener);
+        },
+    };
+};
+/** Run the docpull CLI with the given arguments and capture its output. */
+export function runDocpull(args, options = {}) {
+    const command = options.bin ?? "docpull";
+    const spawnFn = options.spawnFn ?? defaultSpawn;
+    const argv = [...args];
+    return new Promise((resolve, reject) => {
+        const child = spawnFn(command, argv, { cwd: options.cwd, env: options.env, shell: false });
+        let stdout = "";
+        let stderr = "";
+        child.stdout?.on("data", (chunk) => {
+            stdout += String(chunk);
+        });
+        child.stderr?.on("data", (chunk) => {
+            stderr += String(chunk);
+        });
+        child.on("error", (error) => {
+            reject(new DocpullCliError(`Failed to run ${command}: ${error.message}`, null, stderr));
+        });
+        child.on("close", (code) => {
+            if (code === 0) {
+                resolve({ command, args: argv, stdout, stderr, exitCode: 0 });
+                return;
+            }
+            const detail = stderr.trim();
+            reject(new DocpullCliError(`${command} exited with code ${String(code)}${detail ? `: ${detail}` : ""}`, code, stderr));
+        });
+    });
+}
+/** Fetch a URL into a pack directory: `docpull <url> -o <outputDir>`. */
+export function fetchToPack(url, outputDir, options = {}) {
+    if (url.trim().length === 0) {
+        throw new Error("fetchToPack requires a non-empty url");
+    }
+    if (outputDir.trim().length === 0) {
+        throw new Error("fetchToPack requires a non-empty outputDir");
+    }
+    const args = [url, "-o", outputDir];
+    if (options.budget !== undefined) {
+        args.push("--budget", String(options.budget));
+    }
+    if (options.extraArgs !== undefined) {
+        args.push(...options.extraArgs);
+    }
+    return runDocpull(args, options);
+}

package/dist/index.d.ts

@@ -0,0 +1,4 @@
+export { readCorpusManifest, readDocuments, readPack } from "./pack.js";
+export type { CorpusManifest, DocumentRecord, ManifestRecord, Pack } from "./pack.js";
+export { DocpullCliError, fetchToPack, runDocpull } from "./cli.js";
+export type { FetchToPackOptions, RunDocpullOptions, RunDocpullResult, SpawnCallOptions, SpawnedProcess, SpawnedProcessStream, SpawnFunction, } from "./cli.js";

package/dist/index.js

@@ -0,0 +1,2 @@
+export { readCorpusManifest, readDocuments, readPack } from "./pack.js";
+export { DocpullCliError, fetchToPack, runDocpull } from "./cli.js";

package/dist/pack.d.ts

@@ -0,0 +1,84 @@
+/**
+ * Typed readers for the docpull context pack contract (v3).
+ *
+ * A pack directory contains `corpus.manifest.json` (the stable source map)
+ * and `documents.ndjson` or `documents.jsonl` (one document record per
+ * line). Field names mirror the Python `DocumentRecord` model and the
+ * manifest writer; both drop `null` fields on write, so most fields are
+ * optional here. Unknown fields are forward-compatible metadata.
+ */
+/** One entry of `corpus.manifest.json` `records[]`. */
+export interface ManifestRecord {
+    schema_version?: number;
+    document_id: string;
+    url: string;
+    title?: string | null;
+    content_hash: string;
+    fetched_at?: string;
+    rendered_at?: string | null;
+    content_type?: string;
+    mime_type?: string;
+    token_count?: number | null;
+    route?: Record<string, unknown>;
+    rights?: Record<string, unknown>;
+    source_type?: string;
+    source_citation_id?: string;
+    record_citation_id?: string;
+    chunk_index?: number;
+    chunk_id?: string;
+    chunk_heading?: string;
+    output_path?: string;
+    [key: string]: unknown;
+}
+/** Shape of `corpus.manifest.json`. */
+export interface CorpusManifest {
+    schema_version: number;
+    output_contract_version?: number;
+    generated_at?: string;
+    output_format?: string;
+    run?: Record<string, unknown> | null;
+    document_count: number;
+    record_count: number;
+    chunk_count?: number;
+    archive?: Record<string, unknown>;
+    records: ManifestRecord[];
+    [key: string]: unknown;
+}
+/** One line of `documents.ndjson` / `documents.jsonl`. */
+export interface DocumentRecord {
+    schema_version?: number;
+    document_id: string;
+    url: string;
+    title?: string | null;
+    content: string;
+    metadata?: Record<string, unknown>;
+    extraction?: Record<string, unknown>;
+    source_type?: string | null;
+    fetched_at?: string;
+    rendered_at?: string | null;
+    content_type?: string;
+    mime_type?: string;
+    content_hash: string;
+    run?: Record<string, unknown> | null;
+    route?: Record<string, unknown>;
+    rights?: Record<string, unknown>;
+    source_citation_id?: string | null;
+    record_citation_id?: string | null;
+    chunk_index?: number | null;
+    chunk_id?: string | null;
+    chunk_heading?: string | null;
+    token_count?: number | null;
+    [key: string]: unknown;
+}
+/** Manifest plus materialized document records for one pack directory. */
+export interface Pack {
+    packDir: string;
+    manifest: CorpusManifest;
+    documents: DocumentRecord[];
+}
+/** Read and parse `corpus.manifest.json` from a pack directory. */
+export declare function readCorpusManifest(packDir: string): Promise<CorpusManifest>;
+/** Iterate document records from `documents.ndjson` or `documents.jsonl`. */
+export declare function readDocuments(packDir: string): AsyncGenerator<DocumentRecord, void, undefined>;
+/** Read a whole pack: manifest plus every document record. */
+export declare function readPack(packDir: string): Promise<Pack>;

package/dist/pack.js

@@ -0,0 +1,101 @@
+/**
+ * Typed readers for the docpull context pack contract (v3).
+ *
+ * A pack directory contains `corpus.manifest.json` (the stable source map)
+ * and `documents.ndjson` or `documents.jsonl` (one document record per
+ * line). Field names mirror the Python `DocumentRecord` model and the
+ * manifest writer; both drop `null` fields on write, so most fields are
+ * optional here. Unknown fields are forward-compatible metadata.
+ */
+import { createReadStream } from "node:fs";
+import { access, readFile } from "node:fs/promises";
+import { join } from "node:path";
+const DOCUMENT_FILES = ["documents.ndjson", "documents.jsonl"];
+/** Read and parse `corpus.manifest.json` from a pack directory. */
+export async function readCorpusManifest(packDir) {
+    const manifestPath = join(packDir, "corpus.manifest.json");
+    let raw;
+    try {
+        raw = await readFile(manifestPath, "utf8");
+    }
+    catch (error) {
+        throw new Error(`Cannot read corpus manifest at ${manifestPath}: ${errorMessage(error)}`);
+    }
+    let parsed;
+    try {
+        parsed = JSON.parse(raw);
+    }
+    catch (error) {
+        throw new Error(`Invalid JSON in ${manifestPath}: ${errorMessage(error)}`);
+    }
+    if (!isRecord(parsed) || !Array.isArray(parsed["records"])) {
+        throw new Error(`Invalid corpus manifest at ${manifestPath}: expected an object with a records array`);
+    }
+    return parsed;
+}
+/** Iterate document records from `documents.ndjson` or `documents.jsonl`. */
+export async function* readDocuments(packDir) {
+    const documentsPath = await resolveDocumentsPath(packDir);
+    let lineNumber = 0;
+    for await (const line of readLines(documentsPath)) {
+        lineNumber += 1;
+        if (line.trim().length === 0) {
+            continue;
+        }
+        let parsed;
+        try {
+            parsed = JSON.parse(line);
+        }
+        catch (error) {
+            throw new Error(`Invalid NDJSON in ${documentsPath} line ${lineNumber}: ${errorMessage(error)}`);
+        }
+        if (!isRecord(parsed)) {
+            throw new Error(`Invalid NDJSON in ${documentsPath} line ${lineNumber}: expected an object`);
+        }
+        yield parsed;
+    }
+}
+/** Read a whole pack: manifest plus every document record. */
+export async function readPack(packDir) {
+    const manifest = await readCorpusManifest(packDir);
+    const documents = [];
+    for await (const record of readDocuments(packDir)) {
+        documents.push(record);
+    }
+    return { packDir, manifest, documents };
+}
+async function resolveDocumentsPath(packDir) {
+    for (const name of DOCUMENT_FILES) {
+        const candidate = join(packDir, name);
+        try {
+            await access(candidate);
+            return candidate;
+        }
+        catch {
+            // Try the next candidate file name.
+        }
+    }
+    throw new Error(`No documents file found in ${packDir}; expected one of: ${DOCUMENT_FILES.join(", ")}`);
+}
+async function* readLines(path) {
+    const stream = createReadStream(path, { encoding: "utf8" });
+    let buffered = "";
+    for await (const chunk of stream) {
+        buffered += String(chunk);
+        let newlineIndex = buffered.indexOf("\n");
+        while (newlineIndex !== -1) {
+            yield buffered.slice(0, newlineIndex).replace(/\r$/, "");
+            buffered = buffered.slice(newlineIndex + 1);
+            newlineIndex = buffered.indexOf("\n");
+        }
+    }
+    if (buffered.length > 0) {
+        yield buffered.replace(/\r$/, "");
+    }
+}
+function isRecord(value) {
+    return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+function errorMessage(error) {
+    return error instanceof Error ? error.message : String(error);
+}

package/package.json

@@ -0,0 +1,44 @@
+{
+  "name": "@raintree-technology/docpull-sdk",
+  "version": "6.5.0",
+  "description": "TypeScript SDK for reading docpull context packs and running the docpull CLI",
+  "license": "MIT",
+  "type": "module",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js"
+    }
+  },
+  "files": [
+    "dist",
+    "README.md"
+  ],
+  "scripts": {
+    "build": "tsc -p tsconfig.build.json",
+    "test": "bun test",
+    "typecheck": "tsc --noEmit"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/raintree-technology/docpull.git",
+    "directory": "sdk/js"
+  },
+  "author": "Raintree Technology",
+  "keywords": [
+    "docpull",
+    "context-pack",
+    "rag",
+    "sdk"
+  ],
+  "devDependencies": {
+    "@types/node": "^25.9.1",
+    "typescript": "^6.0.3"
+  },
+  "packageManager": "bun@1.3.11",
+  "engines": {
+    "node": "24.x"
+  }
+}