@pdpp/local-collector 0.0.0

package/README.md

@@ -0,0 +1,48 @@
+# @pdpp/local-collector
+
+Publishable PDPP local collector runtime for filesystem-class connectors.
+
+This package is intentionally narrower than `@pdpp/polyfill-connectors`: it
+ships only the local collector runner, the device-exporter client, and bundled
+Claude Code / Codex connector entrypoints. Browser/Patchright-backed connectors
+stay out of this package until each has its own publishability review.
+
+## Usage
+
+```bash
+# @pdpp/local-collector package, npx-launched pdpp-local-collector binary
+npx -y @pdpp/local-collector advertise
+
+# @pdpp/local-collector package, npx-launched pdpp-local-collector binary
+npx -y @pdpp/local-collector enroll \
+  --base-url https://<reference-host> \
+  --code <one-time-code>
+
+# @pdpp/local-collector package, npx-launched pdpp-local-collector binary
+PDPP_LOCAL_DEVICE_ID=<device_id> \
+PDPP_LOCAL_DEVICE_TOKEN=<device_token> \
+PDPP_CONNECTION_ID=<source_instance_id> \
+npx -y @pdpp/local-collector run \
+  --base-url https://<reference-host> \
+  --connector claude_code
+```
+
+The collector sends `X-PDPP-Collector-Protocol` on enrollment and every
+device-exporter request. The reference server rejects incompatible versions
+before persisting records or state.
+
+Install globally if you prefer a persistent binary:
+
+```bash
+# @pdpp/local-collector package, installs the pdpp-local-collector binary
+npm i -g @pdpp/local-collector
+pdpp-local-collector advertise
+```
+
+`device_token` is write-capable for its collector lane. Store it in a secret
+manager or root-readable env file, and do not print it in logs, issues, or
+support transcripts.
+
+For a full operator runbook, including Docker move guidance and troubleshooting
+for `403` after source migration and `409 collector_protocol_mismatch`, see
+[`docs/local-collector.md`](../../docs/local-collector.md).

package/dist/local-collector/bin/pdpp-local-collector.js

@@ -0,0 +1,347 @@
+#!/usr/bin/env node
+import { existsSync } from "node:fs";
+import { basename, dirname, extname, join } from "node:path";
+import { fileURLToPath } from "node:url";
+import { ALLOW_CUSTOM_COMMAND_ENV, CollectorCustomCommandRefusedError, CollectorUsageError, } from "../src/errors.js";
+import { BUNDLED_CONNECTOR_IDS, COLLECTOR_PROTOCOL_VERSION, COLLECTOR_RUNTIME_CAPABILITIES, LocalDeviceOutbox, enrollCollector, getBundledConnector, isMainModule, runCollectorConnector, } from "../src/runner.js";
+const DEFAULT_QUEUE_PATH = join(dirname(fileURLToPath(import.meta.url)), "..", ".pdpp-data", "collector-runner-queue.json");
+const LOCAL_COLLECTOR_PACKAGE_NAME = "@pdpp/local-collector";
+const LOCAL_COLLECTOR_PACKAGE_VERSION = "0.0.0";
+const HELP_TEXT = `pdpp-local-collector — PDPP local collector runner.
+
+Subcommands:
+  advertise                       Print runtime capabilities and protocol version.
+  status                          Print local durable outbox health as JSON.
+          [--queue <path>]
+          [--connection-id <id>]
+  doctor                          Print local durable outbox operator diagnostics as JSON.
+          [--queue <path>]
+          [--connection-id <id>]
+  enroll  --base-url <url>        Exchange a one-time enrollment code for a
+          --code <code>             device id + device token.
+          [--device-label <label>]
+  run     --base-url <url>        Run a bundled filesystem-class connector
+          --connector claude_code|codex
+          --device-id <id>
+          --device-token <token>
+          --connection-id <id>
+          [--streams a,b,c]
+          [--backfill-streams attachments]
+          [--run-id <id>]
+
+Public connectors: ${BUNDLED_CONNECTOR_IDS.join(", ")}.
+Browser-bound connectors stay in the monorepo until each has its own
+publishability review.
+
+See: openspec/changes/publish-pdpp-local-collector/design.md.
+`;
+async function main() {
+    const options = parseArgs(process.argv.slice(2));
+    if (options.command === "advertise") {
+        process.stdout.write(`${JSON.stringify({
+            runtime: COLLECTOR_RUNTIME_CAPABILITIES.id,
+            bindings: [...COLLECTOR_RUNTIME_CAPABILITIES.bindings],
+            collector_protocol_version: COLLECTOR_PROTOCOL_VERSION,
+            bundled_connectors: BUNDLED_CONNECTOR_IDS,
+        }, null, 2)}\n`);
+        return;
+    }
+    if (options.command === "status" || options.command === "doctor") {
+        const status = inspectLocalOutboxStatus(options);
+        process.stdout.write(`${JSON.stringify(options.command === "doctor" ? buildLocalOutboxDoctor(status) : status, null, 2)}\n`);
+        return;
+    }
+    if (options.command === "enroll") {
+        if (!options.code) {
+            throw new CollectorUsageError("enroll requires --code <one-time-code>");
+        }
+        const response = await enrollCollector({
+            baseUrl: options.baseUrl,
+            code: options.code,
+            ...(options.deviceLabel ? { deviceLabel: options.deviceLabel } : {}),
+        });
+        process.stdout.write(`${JSON.stringify(response, null, 2)}\n`);
+        return;
+    }
+    if (!(options.deviceId && options.deviceToken && options.sourceInstanceId)) {
+        throw new CollectorUsageError("run requires --device-id <id>, --device-token <token>, and --connection-id <id>");
+    }
+    if (!options.connector) {
+        throw new CollectorUsageError("run requires --connector <connector-id>");
+    }
+    const spec = buildConnectorSpec(options);
+    const result = await runCollectorConnector({
+        baseUrl: options.baseUrl,
+        connector: spec,
+        deviceId: options.deviceId,
+        deviceToken: options.deviceToken,
+        queuePath: scopedDefaultQueuePath(options.queuePath, DEFAULT_QUEUE_PATH, options.sourceInstanceId),
+        ...(options.runId ? { runId: options.runId } : {}),
+        sourceInstanceId: options.sourceInstanceId,
+    });
+    process.stdout.write(`${JSON.stringify(summarizeRunResultForCli(result), null, 2)}\n`);
+}
+export function summarizeRunResultForCli(result) {
+    return {
+        ...result,
+        flushedState: summarizeCollectorState(result.flushedState),
+        priorState: summarizeCollectorState(result.priorState),
+    };
+}
+function summarizeCollectorState(state) {
+    if (!state || Object.keys(state).length === 0) {
+        return null;
+    }
+    const streams = {};
+    for (const [stream, cursor] of Object.entries(state).sort(([a], [b]) => a.localeCompare(b))) {
+        streams[stream] = summarizeCursor(cursor);
+    }
+    return {
+        stream_count: Object.keys(streams).length,
+        streams,
+    };
+}
+function summarizeCursor(cursor) {
+    if (!cursor || typeof cursor !== "object" || Array.isArray(cursor)) {
+        return { keys: [] };
+    }
+    const record = cursor;
+    const summary = {
+        keys: Object.keys(record).sort(),
+    };
+    if (typeof record.fetched_at === "string") {
+        summary.fetched_at = record.fetched_at;
+    }
+    if (record.file_mtimes && typeof record.file_mtimes === "object" && !Array.isArray(record.file_mtimes)) {
+        summary.file_mtimes_count = Object.keys(record.file_mtimes).length;
+    }
+    return summary;
+}
+export function inspectLocalOutboxStatus(options) {
+    const dbPath = resolveOutboxPath(options);
+    const exists = existsSync(dbPath);
+    const summary = exists ? readOutboxSummary(dbPath, options.sourceInstanceId) : emptyOutboxSummary();
+    return {
+        collector_protocol_version: COLLECTOR_PROTOCOL_VERSION,
+        configured_device: {
+            device_id_configured: Boolean(options.deviceId),
+            device_token_configured: Boolean(options.deviceToken),
+        },
+        db: {
+            configured: Boolean(options.queuePath),
+            exists,
+            path: dbPath,
+        },
+        outbox: {
+            counts: {
+                dead_letter: summary.deadLetter,
+                leased: summary.leased,
+                pending: summary.ready,
+                retrying: summary.retrying,
+                sent: summary.succeeded,
+                total: summary.total,
+            },
+            expired_leases: summary.staleLeases,
+            oldest_pending_at: summary.oldestReadyAt,
+        },
+        package: {
+            name: LOCAL_COLLECTOR_PACKAGE_NAME,
+            version: LOCAL_COLLECTOR_PACKAGE_VERSION,
+        },
+        source: {
+            source_instance_id: options.sourceInstanceId ?? null,
+        },
+    };
+}
+export function buildLocalOutboxDoctor(status) {
+    const checks = {
+        expired_leases: status.outbox.expired_leases > 0 ? "warn" : "ok",
+        outbox_db: status.db.exists ? "ok" : "missing",
+        outbox_failures: status.outbox.counts.dead_letter > 0 ? "fail" : "ok",
+    };
+    return {
+        ...status,
+        checks,
+        status: checks.outbox_failures === "fail"
+            ? "critical"
+            : checks.expired_leases === "warn" || checks.outbox_db === "missing"
+                ? "warning"
+                : "ok",
+    };
+}
+export function buildConnectorSpec(options) {
+    if (!options.connector) {
+        throw new CollectorUsageError("connector required");
+    }
+    const bundled = getBundledConnector(options.connector);
+    const customAllowed = process.env[ALLOW_CUSTOM_COMMAND_ENV] === "1";
+    if (options.entrypointCommand && !customAllowed) {
+        throw new CollectorCustomCommandRefusedError();
+    }
+    if (!(bundled || customAllowed)) {
+        throw new CollectorUsageError(`connector '${options.connector}' is not bundled with pdpp-local-collector. ` +
+            `Supported: ${BUNDLED_CONNECTOR_IDS.join(", ")}. ` +
+            `Set ${ALLOW_CUSTOM_COMMAND_ENV}=1 to use --command <bin> for monorepo development.`);
+    }
+    const command = options.entrypointCommand ?? bundled?.command ?? "tsx";
+    const args = options.args ?? [...(bundled?.args ?? [`connectors/${options.connector}/index.ts`])];
+    const streams = options.streams ?? [...(bundled?.streams ?? [])];
+    if (streams.length === 0) {
+        throw new CollectorUsageError(`run requires --streams <a,b,c> for connector ${options.connector}`);
+    }
+    return {
+        connector_id: options.connector,
+        streams,
+        ...(options.streamsToBackfill ? { streamsToBackfill: options.streamsToBackfill } : {}),
+        command,
+        args,
+        runtime_requirements: { bindings: bundled?.bindings ?? {} },
+    };
+}
+export function parseArgs(args) {
+    const [command, ...rest] = args;
+    if (command === "--help" || command === "-h" || command === "help" || !command) {
+        process.stdout.write(HELP_TEXT);
+        process.exit(0);
+    }
+    if (command !== "enroll" && command !== "run" && command !== "advertise" && command !== "status" && command !== "doctor") {
+        throw new CollectorUsageError(`usage: pdpp-local-collector <enroll|run|advertise|status|doctor> --base-url <url> [options]`);
+    }
+    const options = {
+        baseUrl: process.env.PDPP_REFERENCE_BASE_URL ?? "http://127.0.0.1:7662",
+        command,
+        queuePath: process.env.PDPP_COLLECTOR_QUEUE ?? DEFAULT_QUEUE_PATH,
+    };
+    if (process.env.PDPP_LOCAL_DEVICE_ID) {
+        options.deviceId = process.env.PDPP_LOCAL_DEVICE_ID;
+    }
+    if (process.env.PDPP_LOCAL_DEVICE_TOKEN) {
+        options.deviceToken = process.env.PDPP_LOCAL_DEVICE_TOKEN;
+    }
+    if (process.env.PDPP_COLLECTOR_CONNECTOR) {
+        options.connector = process.env.PDPP_COLLECTOR_CONNECTOR;
+    }
+    if (process.env.PDPP_SOURCE_INSTANCE_ID) {
+        options.sourceInstanceId = process.env.PDPP_SOURCE_INSTANCE_ID;
+    }
+    if (process.env.PDPP_CONNECTION_ID) {
+        options.sourceInstanceId = process.env.PDPP_CONNECTION_ID;
+    }
+    if (process.env.PDPP_RUN_ID) {
+        options.runId = process.env.PDPP_RUN_ID;
+    }
+    for (let index = 0; index < rest.length; index++) {
+        const arg = rest[index];
+        if (!arg) {
+            throw new CollectorUsageError("missing option");
+        }
+        const value = rest[index + 1];
+        applyOption(options, arg, value);
+        index++;
+    }
+    return options;
+}
+function applyOption(options, arg, value) {
+    if (!value) {
+        throw new CollectorUsageError(`missing option value: ${arg}`);
+    }
+    const setters = {
+        "--base-url": (next) => {
+            options.baseUrl = next;
+        },
+        "--backfill-streams": (next) => {
+            options.streamsToBackfill = parseCsv(next);
+        },
+        "--code": (next) => {
+            options.code = next;
+        },
+        "--connector": (next) => {
+            options.connector = next;
+        },
+        "--device-id": (next) => {
+            options.deviceId = next;
+        },
+        "--device-label": (next) => {
+            options.deviceLabel = next;
+        },
+        "--device-token": (next) => {
+            options.deviceToken = next;
+        },
+        "--queue": (next) => {
+            options.queuePath = next;
+        },
+        "--run-id": (next) => {
+            options.runId = next;
+        },
+        "--connection-id": (next) => {
+            options.sourceInstanceId = next;
+        },
+        "--source-instance-id": (next) => {
+            options.sourceInstanceId = next;
+        },
+        "--streams": (next) => {
+            options.streams = parseCsv(next);
+        },
+        "--command": (next) => {
+            options.entrypointCommand = next;
+        },
+        "--args": (next) => {
+            options.args = next.split(" ").filter(Boolean);
+        },
+    };
+    const set = setters[arg];
+    if (!set) {
+        throw new CollectorUsageError(`unknown option: ${arg}`);
+    }
+    set(value);
+}
+function parseCsv(value) {
+    return value
+        .split(",")
+        .map((s) => s.trim())
+        .filter(Boolean);
+}
+export function scopedDefaultQueuePath(queuePath, defaultQueuePath, connectionId) {
+    if (queuePath !== defaultQueuePath) {
+        return queuePath;
+    }
+    const extension = extname(defaultQueuePath);
+    const stem = basename(defaultQueuePath, extension);
+    return join(dirname(defaultQueuePath), `${stem}.${safeQueuePathSegment(connectionId)}${extension}`);
+}
+function resolveOutboxPath(options) {
+    return options.sourceInstanceId
+        ? scopedDefaultQueuePath(options.queuePath, DEFAULT_QUEUE_PATH, options.sourceInstanceId)
+        : options.queuePath;
+}
+function readOutboxSummary(path, sourceInstanceId) {
+    const outbox = new LocalDeviceOutbox({ path });
+    try {
+        return outbox.summary(sourceInstanceId ? { sourceInstanceId } : {});
+    }
+    finally {
+        outbox.close();
+    }
+}
+function emptyOutboxSummary() {
+    return {
+        deadLetter: 0,
+        leased: 0,
+        oldestReadyAt: null,
+        ready: 0,
+        retrying: 0,
+        staleLeases: 0,
+        succeeded: 0,
+        total: 0,
+    };
+}
+function safeQueuePathSegment(value) {
+    return encodeURIComponent(value).replaceAll("%", "_");
+}
+if (isMainModule(import.meta.url)) {
+    main().catch((error) => {
+        process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`);
+        const exitCode = error instanceof CollectorUsageError ? error.exitCode : 1;
+        process.exit(exitCode);
+    });
+}

package/dist/local-collector/src/errors.d.ts

@@ -0,0 +1,12 @@
+export { CollectorStateReadError, RUNTIME_CAPABILITY_MISMATCH_CODE, RuntimeCapabilityMismatchError, } from "../../polyfill-connectors/src/runner/index.js";
+export declare const ALLOW_CUSTOM_COMMAND_ENV = "PDPP_LOCAL_COLLECTOR_ALLOW_CUSTOM_COMMAND";
+export declare class CollectorCustomCommandRefusedError extends Error {
+    readonly code: "custom_command_refused";
+    constructor();
+}
+export declare class CollectorUsageError extends Error {
+    readonly exitCode: number;
+    constructor(message: string, options?: {
+        exitCode?: number;
+    });
+}

package/dist/local-collector/src/errors.js

@@ -0,0 +1,20 @@
+export { CollectorStateReadError, RUNTIME_CAPABILITY_MISMATCH_CODE, RuntimeCapabilityMismatchError, } from "../../polyfill-connectors/src/runner/index.js";
+export const ALLOW_CUSTOM_COMMAND_ENV = "PDPP_LOCAL_COLLECTOR_ALLOW_CUSTOM_COMMAND";
+export class CollectorCustomCommandRefusedError extends Error {
+    code;
+    constructor() {
+        super(`pdpp-local-collector refuses --command <bin> by default to keep the ` +
+            `device-token supply chain narrow. Set ${ALLOW_CUSTOM_COMMAND_ENV}=1 ` +
+            `to opt in for monorepo development; see openspec/changes/publish-pdpp-local-collector/design.md §3.`);
+        this.name = "CollectorCustomCommandRefusedError";
+        this.code = "custom_command_refused";
+    }
+}
+export class CollectorUsageError extends Error {
+    exitCode;
+    constructor(message, options = {}) {
+        super(message);
+        this.name = "CollectorUsageError";
+        this.exitCode = options.exitCode ?? 64;
+    }
+}

package/dist/local-collector/src/runner.d.ts

@@ -0,0 +1,16 @@
+import { type RuntimeCapabilityProfile } from "../../polyfill-connectors/src/runner/index.js";
+export { buildCollectorStartMessage, COLLECTOR_PROTOCOL_VERSION, CollectorStateReadError, drainCollectorQueue, emitToStdout, enrollCollector, evaluatePlacement, isMainModule, LocalDeviceClient, LocalDeviceHttpError, LocalDeviceOutbox, LocalDeviceQueue, PROVIDER_RUNTIME_CAPABILITIES, RUNTIME_CAPABILITY_MISMATCH_CODE, RuntimeCapabilityMismatchError, assertPlacementOrThrow, buildLocalDeviceRecordEnvelope, buildLocalDeviceOutboxId, canonicalJson, diffRequiredBindings, hashCanonicalJson, parseJsonlLine, resourceSet, runCollectorConnector, stringifyForJsonl, transformRecordsToCollectorEnvelopes, type CollectorChildContext, type CollectorConnectorSpec, type CollectorEnrollmentConfig, type CollectorRunConfig, type CollectorRunResult, type ConnectorPlacementInput, type ConnectorRuntimeRequirements, type EmittedMessage, type EnrollmentExchangeResponse, type LocalDeviceRecordEnvelope, type BuildLocalDeviceOutboxIdInput, type LocalDeviceOutboxClaimInput, type LocalDeviceOutboxDeadLetterInput, type LocalDeviceOutboxEnqueueInput, type LocalDeviceOutboxFailInput, type LocalDeviceOutboxItem, type LocalDeviceOutboxKind, type LocalDeviceOutboxLeaseInput, type LocalDeviceOutboxOptions, type LocalDeviceOutboxStatus, type LocalDeviceOutboxSummary, type PlacementDecision, type RuntimeBindingName, type RuntimeCapabilityProfile, type StartMessage, type StreamScope, } from "../../polyfill-connectors/src/runner/index.js";
+export declare const COLLECTOR_RUNTIME_CAPABILITIES: RuntimeCapabilityProfile;
+export interface BundledConnectorEntry {
+    readonly connector_id: string;
+    readonly args: readonly string[];
+    readonly command: string;
+    readonly bindings: Readonly<Record<string, {
+        required: boolean;
+    }>>;
+    readonly streams: readonly string[];
+}
+export declare const BUNDLED_CONNECTORS: Readonly<Record<string, BundledConnectorEntry>>;
+export declare const BUNDLED_CONNECTOR_IDS: readonly string[];
+export declare function getBundledConnector(connectorId: string): BundledConnectorEntry | null;
+export declare const BUNDLED_CONNECTOR_VERSIONS: Readonly<Record<string, string>>;

package/dist/local-collector/src/runner.js

@@ -0,0 +1,59 @@
+import { existsSync } from "node:fs";
+import { extname } from "node:path";
+import { fileURLToPath } from "node:url";
+import { COLLECTOR_PROTOCOL_VERSION as PROTOCOL_VERSION, COLLECTOR_RUNTIME_CAPABILITIES as POLYFILL_COLLECTOR_RUNTIME_CAPABILITIES, } from "../../polyfill-connectors/src/runner/index.js";
+export { buildCollectorStartMessage, COLLECTOR_PROTOCOL_VERSION, CollectorStateReadError, drainCollectorQueue, emitToStdout, enrollCollector, evaluatePlacement, isMainModule, LocalDeviceClient, LocalDeviceHttpError, LocalDeviceOutbox, LocalDeviceQueue, PROVIDER_RUNTIME_CAPABILITIES, RUNTIME_CAPABILITY_MISMATCH_CODE, RuntimeCapabilityMismatchError, assertPlacementOrThrow, buildLocalDeviceRecordEnvelope, buildLocalDeviceOutboxId, canonicalJson, diffRequiredBindings, hashCanonicalJson, parseJsonlLine, resourceSet, runCollectorConnector, stringifyForJsonl, transformRecordsToCollectorEnvelopes, } from "../../polyfill-connectors/src/runner/index.js";
+export const COLLECTOR_RUNTIME_CAPABILITIES = {
+    id: POLYFILL_COLLECTOR_RUNTIME_CAPABILITIES.id,
+    bindings: new Set(["network", "filesystem", "local_device"]),
+};
+function bundledEntry(connectorPath) {
+    const built = fileURLToPath(new URL(`../../polyfill-connectors/connectors/${connectorPath}/index.js`, import.meta.url));
+    if (existsSync(built)) {
+        return built;
+    }
+    return fileURLToPath(new URL(`../../polyfill-connectors/connectors/${connectorPath}/index.ts`, import.meta.url));
+}
+function commandForEntry(entry) {
+    return extname(entry) === ".ts" ? "tsx" : "node";
+}
+const POLYFILL_CLAUDE_CODE_ENTRY = bundledEntry("claude_code");
+const POLYFILL_CODEX_ENTRY = bundledEntry("codex");
+export const BUNDLED_CONNECTORS = Object.freeze({
+    claude_code: Object.freeze({
+        connector_id: "claude_code",
+        command: commandForEntry(POLYFILL_CLAUDE_CODE_ENTRY),
+        args: Object.freeze([POLYFILL_CLAUDE_CODE_ENTRY]),
+        bindings: Object.freeze({ filesystem: Object.freeze({ required: true }) }),
+        streams: Object.freeze([
+            "sessions",
+            "messages",
+            "attachments",
+            "memory_notes",
+            "skills",
+            "slash_commands",
+        ]),
+    }),
+    codex: Object.freeze({
+        connector_id: "codex",
+        command: commandForEntry(POLYFILL_CODEX_ENTRY),
+        args: Object.freeze([POLYFILL_CODEX_ENTRY]),
+        bindings: Object.freeze({ filesystem: Object.freeze({ required: true }) }),
+        streams: Object.freeze([
+            "sessions",
+            "messages",
+            "function_calls",
+            "rules",
+            "prompts",
+            "skills",
+        ]),
+    }),
+});
+export const BUNDLED_CONNECTOR_IDS = Object.freeze(Object.keys(BUNDLED_CONNECTORS));
+export function getBundledConnector(connectorId) {
+    return BUNDLED_CONNECTORS[connectorId] ?? null;
+}
+export const BUNDLED_CONNECTOR_VERSIONS = Object.freeze({
+    claude_code: PROTOCOL_VERSION,
+    codex: PROTOCOL_VERSION,
+});