@pdpp/local-collector 0.0.0

package/dist/polyfill-connectors/src/scope-filters.js

@@ -0,0 +1,80 @@
+export function resourceSet(streamRequest) {
+    if (!(streamRequest && Array.isArray(streamRequest.resources) && streamRequest.resources.length)) {
+        return null;
+    }
+    const s = new Set();
+    for (const r of streamRequest.resources) {
+        s.add(String(r));
+    }
+    return s;
+}
+export function passesResourceFilter(resSet, primaryKey) {
+    if (!resSet) {
+        return true;
+    }
+    const canonical = Array.isArray(primaryKey) ? JSON.stringify(primaryKey.map(String)) : String(primaryKey);
+    return resSet.has(canonical);
+}
+export function passesTimeRange(isoValue, timeRange) {
+    if (!timeRange) {
+        return true;
+    }
+    if (!isoValue) {
+        return true;
+    }
+    if (timeRange.since && isoValue < timeRange.since) {
+        return false;
+    }
+    if (timeRange.until && isoValue >= timeRange.until) {
+        return false;
+    }
+    return true;
+}
+export function makeEmitGate(emitRecord, streamRequest, { consentTimeField } = {}) {
+    const resSet = resourceSet(streamRequest);
+    const emitted = new Set();
+    const gate = ((stream, data, keyField = "id") => {
+        const key = data[keyField];
+        if (key == null) {
+            return false;
+        }
+        const canonical = Array.isArray(key) ? JSON.stringify(key.map(String)) : String(key);
+        if (resSet && !resSet.has(canonical)) {
+            return false;
+        }
+        if (consentTimeField && streamRequest?.time_range) {
+            const v = data[consentTimeField];
+            const iso = typeof v === "string" ? v : undefined;
+            if (!passesTimeRange(iso, streamRequest.time_range)) {
+                return false;
+            }
+        }
+        emitted.add(canonical);
+        emitRecord(stream, data);
+        return true;
+    });
+    gate.emittedSet = () => emitted;
+    return gate;
+}
+export function emitTombstones({ emit, stream, priorIds, currentIds, emittedAt }) {
+    let count = 0;
+    for (const id of priorIds || []) {
+        if (!currentIds.has(id)) {
+            emit({
+                type: "RECORD",
+                stream,
+                key: id,
+                data: { id },
+                emitted_at: emittedAt,
+                op: "delete",
+            });
+            count++;
+        }
+    }
+    return count;
+}
+export async function requireCredentialsOrAsk({ required, connectorName, sendInteraction, }) {
+    const { resolveAuth } = await import("./auth.js");
+    const ctx = { sendInteraction, connectorName };
+    return resolveAuth({ kind: "env", required }, ctx);
+}

package/dist/polyfill-connectors/src/shutdown-hook.js

@@ -0,0 +1,51 @@
+const SIGNALS_TO_HOOK = ["SIGTERM", "SIGINT"];
+const SIGNAL_EXIT_CODES = {
+    SIGTERM: 128 + 15,
+    SIGINT: 128 + 2,
+};
+function writeShutdownStderr(message) {
+    try {
+        process.stderr.write(message);
+    }
+    catch {
+    }
+}
+async function runStepSwallowing(fn, label, signal) {
+    if (!fn) {
+        return;
+    }
+    try {
+        await fn();
+    }
+    catch (err) {
+        writeShutdownStderr(`[shutdown-hook] ${label}() rejected during ${signal}: ${err instanceof Error ? err.message : String(err)}\n`);
+    }
+}
+export function withShutdownRelease(release, options = {}) {
+    let firing = false;
+    const { finalize } = options;
+    const handle = (signal) => async () => {
+        if (firing) {
+            return;
+        }
+        firing = true;
+        try {
+            await runStepSwallowing(finalize, "finalize", signal);
+            await runStepSwallowing(release, "release", signal);
+        }
+        finally {
+            process.exit(SIGNAL_EXIT_CODES[signal]);
+        }
+    };
+    const listeners = [];
+    for (const signal of SIGNALS_TO_HOOK) {
+        const fn = handle(signal);
+        process.on(signal, fn);
+        listeners.push({ signal, fn });
+    }
+    return function dispose() {
+        for (const { signal, fn } of listeners) {
+            process.removeListener(signal, fn);
+        }
+    };
+}

package/dist/polyfill-connectors/src/streaming-target-registration.js

@@ -0,0 +1,161 @@
+const REGISTRATION_PATH = (runId, interactionId) => `/admin/runs/${encodeURIComponent(runId)}/interactions/${encodeURIComponent(interactionId)}/streaming-target`;
+const ALLOWED_CDP_TARGET_HOSTS = new Set(["127.0.0.1", "localhost", "neko"]);
+const defaultLogger = {
+    warn(message, data) {
+        const suffix = data ? ` ${JSON.stringify(data)}` : "";
+        process.stderr.write(`[streaming-registration] ${message}${suffix}\n`);
+    },
+};
+function isAllowedCdpWsUrl(wsUrl) {
+    let parsed;
+    try {
+        parsed = new URL(wsUrl);
+    }
+    catch {
+        return false;
+    }
+    if (parsed.protocol !== "ws:" && parsed.protocol !== "wss:") {
+        return false;
+    }
+    return ALLOWED_CDP_TARGET_HOSTS.has(parsed.hostname);
+}
+function isDescriptorObject(value) {
+    return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+function addOptionalMetadata(body, { pageUrl, pageTitle, reason }) {
+    if (typeof pageUrl === "string" && pageUrl.length > 0) {
+        body.page_url = pageUrl;
+    }
+    if (typeof pageTitle === "string" && pageTitle.length > 0) {
+        body.page_title = pageTitle;
+    }
+    if (typeof reason === "string" && reason.length > 0) {
+        body.reason = reason;
+    }
+}
+export function createRegistrationClient(options) {
+    if (!options.baseUrl) {
+        throw new Error("createRegistrationClient: baseUrl required");
+    }
+    if (!options.deviceToken) {
+        throw new Error("createRegistrationClient: deviceToken required");
+    }
+    const baseUrl = new URL(options.baseUrl);
+    const fetchImpl = options.fetch ?? globalThis.fetch;
+    const logger = options.logger ?? defaultLogger;
+    const authHeader = `Bearer ${options.deviceToken}`;
+    if (typeof fetchImpl !== "function") {
+        throw new Error("createRegistrationClient: no fetch implementation available");
+    }
+    return {
+        async register(args) {
+            const { runId, interactionId } = args;
+            if (!runId) {
+                logger.warn("register skipped: runId is empty");
+                return false;
+            }
+            if (!interactionId) {
+                logger.warn("register skipped: interactionId is empty", { runId });
+                return false;
+            }
+            let method;
+            let body;
+            if (args.backend === "neko") {
+                if (!isDescriptorObject(args.descriptor)) {
+                    logger.warn("register skipped: neko descriptor is not an object", { runId, interactionId });
+                    return false;
+                }
+                method = "POST";
+                body = { backend: "neko", descriptor: args.descriptor };
+            }
+            else {
+                const { wsUrl } = args;
+                if (!isAllowedCdpWsUrl(wsUrl)) {
+                    logger.warn("register skipped: wsUrl host is not allowed", { runId, interactionId });
+                    return false;
+                }
+                method = "PUT";
+                body = { ws_url: wsUrl };
+            }
+            addOptionalMetadata(body, args);
+            let response;
+            try {
+                response = await fetchImpl(new URL(REGISTRATION_PATH(runId, interactionId), baseUrl), {
+                    method,
+                    headers: {
+                        accept: "application/json",
+                        authorization: authHeader,
+                        "content-type": "application/json",
+                    },
+                    body: JSON.stringify(body),
+                });
+            }
+            catch (err) {
+                const message = err instanceof Error ? err.message : String(err);
+                logger.warn("register failed (network error)", { runId, interactionId, error: message });
+                return false;
+            }
+            if (!response.ok) {
+                await response.text().catch(() => undefined);
+                logger.warn("register failed", { runId, interactionId, status: response.status });
+                return false;
+            }
+            await response.text().catch(() => undefined);
+            return true;
+        },
+        async unregister({ runId, interactionId }) {
+            if (!(runId && interactionId)) {
+                return false;
+            }
+            let response;
+            try {
+                response = await fetchImpl(new URL(REGISTRATION_PATH(runId, interactionId), baseUrl), {
+                    method: "DELETE",
+                    headers: {
+                        accept: "application/json",
+                        authorization: authHeader,
+                    },
+                });
+            }
+            catch (err) {
+                const message = err instanceof Error ? err.message : String(err);
+                logger.warn("unregister failed (network error)", { runId, interactionId, error: message });
+                return false;
+            }
+            if (!response.ok) {
+                await response.text().catch(() => undefined);
+                if (response.status !== 404) {
+                    logger.warn("unregister failed", { runId, interactionId, status: response.status });
+                }
+                return false;
+            }
+            await response.text().catch(() => undefined);
+            return true;
+        },
+    };
+}
+export { ALLOWED_CDP_TARGET_HOSTS, REGISTRATION_PATH };
+const STREAMING_REGISTRATION_TOKEN_ENV = "PDPP_STREAMING_REGISTRATION_TOKEN";
+const LOCAL_DEVICE_TOKEN_ENV = "PDPP_LOCAL_DEVICE_TOKEN";
+export function resolveStreamingRegistrationFromEnv(env = process.env) {
+    const runId = env.PDPP_RUN_ID?.trim();
+    const baseUrl = env.PDPP_REFERENCE_BASE_URL?.trim();
+    const registrationToken = env[STREAMING_REGISTRATION_TOKEN_ENV]?.trim();
+    const deviceToken = env[LOCAL_DEVICE_TOKEN_ENV]?.trim();
+    const bearerToken = registrationToken || deviceToken;
+    if (!(runId && baseUrl && bearerToken)) {
+        if (runId && !(baseUrl && bearerToken)) {
+            process.stderr.write("[streaming-registration] PDPP_RUN_ID set but PDPP_REFERENCE_BASE_URL or " +
+                `${STREAMING_REGISTRATION_TOKEN_ENV}/${LOCAL_DEVICE_TOKEN_ENV} missing; ` +
+                "streaming-companion target not registered for this run.\n");
+        }
+        return Promise.resolve(undefined);
+    }
+    const client = createRegistrationClient({ baseUrl, deviceToken: bearerToken });
+    return Promise.resolve({
+        runId,
+        register: (args) => client.register(args),
+        unregister: (args) => client.unregister(args),
+    });
+}
+export { LOCAL_DEVICE_TOKEN_ENV, STREAMING_REGISTRATION_TOKEN_ENV };

package/package.json

@@ -0,0 +1,63 @@
+{
+  "name": "@pdpp/local-collector",
+  "version": "0.0.0",
+  "description": "Publishable local collector runtime for PDPP: filesystem-class connectors (Claude Code, Codex) plus the device-exporter ingest client.",
+  "type": "module",
+  "private": false,
+  "main": "./dist/local-collector/src/runner.js",
+  "types": "./dist/local-collector/src/runner.d.ts",
+  "bin": {
+    "pdpp-local-collector": "dist/local-collector/bin/pdpp-local-collector.js"
+  },
+  "exports": {
+    "./runner": {
+      "types": "./dist/local-collector/src/runner.d.ts",
+      "import": "./dist/local-collector/src/runner.js"
+    },
+    "./errors": {
+      "types": "./dist/local-collector/src/errors.d.ts",
+      "import": "./dist/local-collector/src/errors.js"
+    },
+    "./package.json": "./package.json"
+  },
+  "files": [
+    "dist/",
+    "README.md"
+  ],
+  "scripts": {
+    "build": "rm -rf dist && tsc -p tsconfig.build.json && node scripts/postbuild.mjs",
+    "prepack": "pnpm build",
+    "test": "node --test --import tsx 'test/*.test.js'",
+    "pack-install-run": "node scripts/pack-install-run.mjs",
+    "validate:package": "pnpm build && node scripts/validate-package.mjs",
+    "verify": "pnpm test && pnpm validate:package"
+  },
+  "dependencies": {
+    "zod": "^4.3.6"
+  },
+  "keywords": [
+    "pdpp",
+    "personal-data",
+    "collector",
+    "data-portability"
+  ],
+  "license": "ISC",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/vana-com/pdpp.git",
+    "directory": "packages/local-collector"
+  },
+  "bugs": {
+    "url": "https://github.com/vana-com/pdpp/issues"
+  },
+  "homepage": "https://github.com/vana-com/pdpp#readme",
+  "engines": {
+    "node": ">=22.14.0"
+  },
+  "publishConfig": {
+    "access": "public",
+    "provenance": false,
+    "registry": "https://registry.npmjs.org/",
+    "tag": "beta"
+  }
+}