@pdpp/local-collector 0.0.0

package/dist/polyfill-connectors/connectors/codex/parsers.js

@@ -0,0 +1,159 @@
+import { PDPP_PREVIEW_MAX_CHARS, safeTextPreview } from "../../src/safe-text-preview.js";
+const FRONTMATTER_RE = /^---\s*\r?\n([\s\S]*?)\r?\n---\s*\r?\n?([\s\S]*)$/;
+export const YEAR_DIR_RE = /^\d{4}$/;
+export const TWO_DIGIT_DIR_RE = /^\d{2}$/;
+const LINE_SPLIT_RE = /\r?\n/;
+const FRONTMATTER_KV_RE = /^([A-Za-z0-9_-]+)\s*:\s*(.*)$/;
+export const RULES_SUFFIX_RE = /\.rules$/;
+export const MD_SUFFIX_RE = /\.md$/;
+export function textPreview(s, max = PDPP_PREVIEW_MAX_CHARS) {
+    const r = safeTextPreview(s, max);
+    return r.preview;
+}
+export function extractMessageText(payload) {
+    if (!(payload?.content && Array.isArray(payload.content))) {
+        return null;
+    }
+    const parts = payload.content.map((p) => p?.text).filter(Boolean);
+    return parts.join("\n") || null;
+}
+export function payloadOutputPreview(output, max = PDPP_PREVIEW_MAX_CHARS) {
+    let toPreview = output;
+    if (typeof output !== "string" && output !== null && output !== undefined) {
+        toPreview = JSON.stringify(output);
+    }
+    const r = safeTextPreview(toPreview, max);
+    return {
+        preview: r.preview,
+        binaryReason: r.kind === "binary" ? r.reason : null,
+    };
+}
+export function epochToIso(sec) {
+    return Number.isFinite(sec) && typeof sec === "number" && sec > 0 ? new Date(sec * 1000).toISOString() : null;
+}
+function stripSurroundingQuotes(value) {
+    if ((value.startsWith('"') && value.endsWith('"')) || (value.startsWith("'") && value.endsWith("'"))) {
+        return value.slice(1, -1);
+    }
+    return value;
+}
+function parseFrontmatterLine(line, meta) {
+    const kv = line.match(FRONTMATTER_KV_RE);
+    if (!kv) {
+        return;
+    }
+    const key = kv[1];
+    if (!key) {
+        return;
+    }
+    meta[key] = stripSurroundingQuotes((kv[2] ?? "").trim());
+}
+export function parseFrontmatter(text) {
+    const m = text.match(FRONTMATTER_RE);
+    if (!m) {
+        return { meta: {}, body: text };
+    }
+    const meta = {};
+    for (const line of (m[1] ?? "").split(LINE_SPLIT_RE)) {
+        parseFrontmatterLine(line, meta);
+    }
+    return { meta, body: m[2] ?? "" };
+}
+export function isRolloutFile(name) {
+    return name.startsWith("rollout-") && name.endsWith(".jsonl");
+}
+export function buildThreadSessionRecord(id, t, agg) {
+    return {
+        id,
+        cwd: t.cwd || null,
+        originator: t.source || null,
+        cli_version: t.cli_version || null,
+        model_provider: t.model_provider || null,
+        git_commit: t.git_sha || null,
+        git_branch: t.git_branch || null,
+        repository_url: t.git_origin_url || null,
+        started_at: epochToIso(t.created_at) || agg?.meta?.timestamp || agg?.firstTs || null,
+        last_event_at: epochToIso(t.updated_at) || agg?.lastTs || null,
+        message_count: agg?.messageCount ?? null,
+        function_call_count: agg?.functionCallCount ?? null,
+        title: textPreview(t.title || null, 500),
+        archived: t.archived === 1 || t.archived === true,
+        tokens_used: t.tokens_used ?? null,
+        first_user_message: textPreview(t.first_user_message || null, 2000),
+        sandbox_policy: t.sandbox_policy || null,
+        approval_mode: t.approval_mode || null,
+        rollout_path: t.rollout_path || agg?.rolloutPath || null,
+    };
+}
+export function buildRolloutOnlySessionRecord(id, agg) {
+    const meta = agg.meta || {};
+    return {
+        id,
+        cwd: meta.cwd || null,
+        originator: meta.originator || null,
+        cli_version: meta.cli_version || null,
+        model_provider: meta.model_provider || null,
+        git_commit: meta.git?.commit_hash || null,
+        git_branch: meta.git?.branch || null,
+        repository_url: meta.git?.repository_url || null,
+        started_at: meta.timestamp || agg.firstTs,
+        last_event_at: agg.lastTs,
+        message_count: agg.messageCount,
+        function_call_count: agg.functionCallCount,
+        title: null,
+        archived: null,
+        tokens_used: null,
+        first_user_message: null,
+        sandbox_policy: null,
+        approval_mode: null,
+        rollout_path: agg.rolloutPath || null,
+    };
+}
+export function splitRulesLines(text) {
+    return text.split(LINE_SPLIT_RE);
+}
+export function isSkippableRulesLine(line) {
+    return !line || line.startsWith("#");
+}
+export function buildRuleRecord(args) {
+    return {
+        id: `rules:${args.ruleset}:${args.index}`,
+        ruleset: args.ruleset,
+        rule_text: textPreview(args.line, 4000),
+        rule_index: args.index,
+        path: args.path,
+        mtime_epoch: args.mtime,
+    };
+}
+export function buildPromptRecord(args) {
+    const name = args.meta.name || args.fileName.replace(MD_SUFFIX_RE, "");
+    return {
+        id: `prompts:${args.fileName}`,
+        name,
+        description: args.meta.description || null,
+        content: textPreview(args.body, 20_000),
+        path: args.path,
+        mtime_epoch: Math.floor(args.mtimeMs / 1000),
+    };
+}
+export function buildSkillRecord(args) {
+    return {
+        id: `skills:${args.dirName}`,
+        name: args.meta.name || args.dirName,
+        description: args.meta.description || null,
+        content: textPreview(args.body, 20_000),
+        path: args.path,
+        mtime_epoch: Math.floor(args.mtimeMs / 1000),
+    };
+}
+export function extendTimestampRange(range, ts) {
+    if (!ts) {
+        return;
+    }
+    if (!range.firstTs || ts < range.firstTs) {
+        range.firstTs = ts;
+    }
+    if (!range.lastTs || ts > range.lastTs) {
+        range.lastTs = ts;
+    }
+}

package/dist/polyfill-connectors/connectors/codex/schemas.js

@@ -0,0 +1,118 @@
+import { z } from "zod";
+import { pdppSafeText } from "../../src/pdpp-safe-text.js";
+import { PDPP_PREVIEW_MAX_CHARS, safeTextPreview } from "../../src/safe-text-preview.js";
+import { makeValidateRecord } from "../../src/schema-registry.js";
+const SESSION_ID_RE = /^[0-9a-f-]{36}$|^[0-9a-z]{26}$/;
+const MESSAGE_ID_RE = /^([0-9a-f-]{36}|[0-9a-z]{26})(:\d+)?(:output)?$|^call_[A-Za-z0-9]{24}$/;
+const FUNCTION_CALL_ID_RE = /^call_[A-Za-z0-9]{24}$|^[0-9a-f-]{36}(:\d+)?(:output)?$|^[0-9a-z]{26}(:\d+)?(:output)?$/;
+const ISO_Z_RE = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/;
+const isoDateTimeSchema = z.string().regex(ISO_Z_RE, "must be ISO-8601 with millis and Z suffix").nullable();
+const nullableBoolSchema = z.boolean().nullable();
+const nullableIntSchema = z.number().int().nullable();
+export const sessionsSchema = z.object({
+    id: z.string().regex(SESSION_ID_RE, "session id must be uuid or ulid"),
+    cwd: pdppSafeText.nullable(),
+    originator: pdppSafeText.nullable(),
+    cli_version: pdppSafeText.nullable(),
+    model_provider: pdppSafeText.nullable(),
+    git_commit: pdppSafeText.nullable(),
+    git_branch: pdppSafeText.nullable(),
+    repository_url: pdppSafeText.nullable(),
+    started_at: isoDateTimeSchema,
+    last_event_at: isoDateTimeSchema,
+    message_count: nullableIntSchema,
+    function_call_count: nullableIntSchema,
+    title: pdppSafeText.max(1_000_000).nullable(),
+    archived: nullableBoolSchema,
+    tokens_used: nullableIntSchema,
+    first_user_message: pdppSafeText.max(1_000_000).nullable(),
+    sandbox_policy: pdppSafeText.nullable(),
+    approval_mode: pdppSafeText.nullable(),
+    rollout_path: pdppSafeText.nullable(),
+});
+export const messagesSchema = z.object({
+    id: z.string().regex(MESSAGE_ID_RE, "message id must be uuid or uuid:line or uuid:line:output"),
+    session_id: z.string().regex(SESSION_ID_RE, "session_id must be uuid or ulid"),
+    role: pdppSafeText.nullable(),
+    type: pdppSafeText.nullable(),
+    content: pdppSafeText.max(10_000_000).nullable(),
+    timestamp: isoDateTimeSchema,
+});
+export const functionCallsSchema = z.object({
+    id: z.string().regex(FUNCTION_CALL_ID_RE, "function_call id must be call_* or uuid or composite"),
+    session_id: z.string().regex(SESSION_ID_RE, "session_id must be uuid or ulid"),
+    call_id: pdppSafeText.nullable(),
+    name: pdppSafeText.nullable(),
+    arguments: pdppSafeText.max(10_000_000).nullable(),
+    output_preview: z
+        .string()
+        .max(PDPP_PREVIEW_MAX_CHARS + 1)
+        .refine((val) => {
+        const result = safeTextPreview(val, PDPP_PREVIEW_MAX_CHARS);
+        return result.kind === "text" || result.kind === "empty";
+    }, "output_preview contains forbidden control characters")
+        .nullable(),
+    output_binary_reason: pdppSafeText.max(200).nullable().optional(),
+    timestamp: isoDateTimeSchema,
+});
+export const rulesSchema = z.object({
+    id: pdppSafeText,
+    ruleset: pdppSafeText,
+    rule_text: pdppSafeText.max(4000),
+    rule_index: z.number().int().min(0),
+    path: pdppSafeText.nullable(),
+    mtime_epoch: z.number().int().nullable(),
+});
+export const promptsSchema = z.object({
+    id: pdppSafeText,
+    name: pdppSafeText,
+    description: pdppSafeText.nullable(),
+    content: pdppSafeText.max(10_000_000).nullable(),
+    path: pdppSafeText.nullable(),
+    mtime_epoch: z.number().int().nullable(),
+});
+export const skillsSchema = z.object({
+    id: pdppSafeText,
+    name: pdppSafeText,
+    description: pdppSafeText.nullable(),
+    content: pdppSafeText.max(10_000_000).nullable(),
+    path: pdppSafeText.nullable(),
+    mtime_epoch: z.number().int().nullable(),
+});
+const inventoryClassificationSchema = z.enum(["inventory_only", "defer"]);
+const inventoryTypeSchema = z.enum(["directory", "file", "missing", "other"]);
+const coverageStatusSchema = z.enum(["collected", "inventory_only", "excluded", "deferred", "missing", "unsupported"]);
+export const inventorySchema = z.object({
+    id: pdppSafeText,
+    store: pdppSafeText,
+    relative_path: pdppSafeText.max(2048),
+    path_hash: z.string().regex(/^[a-f0-9]{64}$/),
+    type: inventoryTypeSchema,
+    size_bytes: z.number().int().min(0).nullable(),
+    mtime_epoch: z.number().int().min(0).nullable(),
+    classification: inventoryClassificationSchema,
+    reason: pdppSafeText.max(512),
+});
+export const coverageDiagnosticsSchema = z.object({
+    id: pdppSafeText,
+    store: pdppSafeText,
+    stream: pdppSafeText.nullable(),
+    status: coverageStatusSchema,
+    reason: pdppSafeText.max(512),
+});
+export const SCHEMAS = {
+    sessions: sessionsSchema,
+    messages: messagesSchema,
+    function_calls: functionCallsSchema,
+    rules: rulesSchema,
+    prompts: promptsSchema,
+    skills: skillsSchema,
+    history: inventorySchema,
+    session_index: inventorySchema,
+    logs: inventorySchema,
+    shell_snapshots: inventorySchema,
+    config_inventory: inventorySchema,
+    cache_inventory: inventorySchema,
+    coverage_diagnostics: coverageDiagnosticsSchema,
+};
+export const validateRecord = makeValidateRecord(SCHEMAS);

package/dist/polyfill-connectors/connectors/codex/types.js

@@ -0,0 +1 @@
+export {};

package/dist/polyfill-connectors/src/auth.js

@@ -0,0 +1,76 @@
+const strategies = new Map();
+export function registerAuthStrategy(kind, resolver) {
+    strategies.set(kind, resolver);
+}
+export function hasAuthStrategy(kind) {
+    return strategies.has(kind);
+}
+export function resolveAuth(config, runtime) {
+    if (!config) {
+        return Promise.resolve({});
+    }
+    const resolver = strategies.get(config.kind);
+    if (!resolver) {
+        return Promise.reject(new Error(`auth_strategy_unknown: ${config.kind}`));
+    }
+    return resolver(config, runtime);
+}
+const SECRET_NAME = /PASSWORD|SECRET|TOKEN/i;
+function resolveEnvEntry(entry) {
+    const aliases = Array.isArray(entry) ? entry : [entry];
+    const primary = aliases[0];
+    if (!primary) {
+        return null;
+    }
+    for (const name of aliases) {
+        const candidate = process.env[name];
+        if (candidate) {
+            return { primary, value: candidate };
+        }
+    }
+    return { primary, value: undefined };
+}
+function buildCredentialSchema(missing, connectorName) {
+    const properties = {};
+    for (const name of missing) {
+        const base = {
+            type: "string",
+            description: `${name} for ${connectorName}`,
+        };
+        properties[name] = SECRET_NAME.test(name) ? { ...base, format: "password" } : base;
+    }
+    return { type: "object", properties, required: missing };
+}
+registerAuthStrategy("env", async (config, runtime) => {
+    const { required } = config;
+    if (!Array.isArray(required) || required.length === 0) {
+        throw new Error("auth_env_required_missing: auth.required must be a non-empty array");
+    }
+    const have = {};
+    const missing = [];
+    for (const entry of required) {
+        const resolved = resolveEnvEntry(entry);
+        if (!resolved) {
+            continue;
+        }
+        if (resolved.value === undefined) {
+            missing.push(resolved.primary);
+        }
+        else {
+            have[resolved.primary] = resolved.value;
+        }
+    }
+    if (missing.length === 0) {
+        return have;
+    }
+    const resp = await runtime.sendInteraction({
+        kind: "credentials",
+        message: `${runtime.connectorName} needs: ${missing.join(", ")}. Set in .env.local for persistence.`,
+        schema: buildCredentialSchema(missing, runtime.connectorName),
+        timeout_seconds: 1800,
+    });
+    if (resp.status !== "success" || !resp.data) {
+        throw new Error(`${runtime.connectorName}_credentials_missing`);
+    }
+    return { ...have, ...resp.data };
+});

package/dist/polyfill-connectors/src/browser-handoff.js

@@ -0,0 +1,197 @@
+import { randomBytes } from "node:crypto";
+import { resolveStreamingRegistrationFromEnv, } from "./streaming-target-registration.js";
+export async function resolveWsUrlForExactPage(page, opts) {
+    const session = await page.context().newCDPSession(page);
+    try {
+        const { targetInfo } = (await session.send("Target.getTargetInfo"));
+        if (targetInfo.type !== "page") {
+            throw new Error(`expected page target, got type=${targetInfo.type}`);
+        }
+        return `ws://${opts.host}:${String(opts.port)}/devtools/page/${targetInfo.targetId}`;
+    }
+    finally {
+        await session.detach().catch(() => undefined);
+    }
+}
+const BROWSER_CDP_HOST_ENV = "PDPP_BROWSER_CDP_HOST";
+const BROWSER_CDP_PORT_ENV = "PDPP_BROWSER_CDP_PORT";
+const BROWSER_SURFACE_ID_ENV = "PDPP_BROWSER_SURFACE_ID";
+const BROWSER_SURFACE_LEASE_ID_ENV = "PDPP_BROWSER_SURFACE_LEASE_ID";
+const BROWSER_SURFACE_PROFILE_KEY_ENV = "PDPP_BROWSER_SURFACE_PROFILE_KEY";
+const BROWSER_SURFACE_REQUIRED_ENV = "PDPP_BROWSER_SURFACE_REQUIRED";
+const BROWSER_SURFACE_STREAM_BASE_URL_ENV = "PDPP_BROWSER_SURFACE_STREAM_BASE_URL";
+function resolveCdpEndpointFromEnv(env) {
+    const host = env[BROWSER_CDP_HOST_ENV]?.trim();
+    const portRaw = env[BROWSER_CDP_PORT_ENV]?.trim();
+    if (!(host && portRaw)) {
+        return;
+    }
+    const port = Number.parseInt(portRaw, 10);
+    if (!(Number.isFinite(port) && port > 0)) {
+        return;
+    }
+    return { host, port };
+}
+function nonEmptyEnv(env, key) {
+    const value = env[key]?.trim();
+    return value ? value : undefined;
+}
+function isManagedNekoRequired(env) {
+    return nonEmptyEnv(env, BROWSER_SURFACE_REQUIRED_ENV)?.toLowerCase() === "neko";
+}
+function resolveManagedNekoDescriptorFromEnv(env) {
+    const baseUrl = nonEmptyEnv(env, BROWSER_SURFACE_STREAM_BASE_URL_ENV);
+    const leaseId = nonEmptyEnv(env, BROWSER_SURFACE_LEASE_ID_ENV);
+    const profileKey = nonEmptyEnv(env, BROWSER_SURFACE_PROFILE_KEY_ENV);
+    if (!(baseUrl && leaseId && profileKey)) {
+        return;
+    }
+    return {
+        backend: "neko",
+        base_url: baseUrl,
+        lease_id: leaseId,
+        profile_key: profileKey,
+        ...(nonEmptyEnv(env, BROWSER_SURFACE_ID_ENV) ? { surface_id: nonEmptyEnv(env, BROWSER_SURFACE_ID_ENV) } : {}),
+    };
+}
+function generateInteractionId() {
+    return `int_${String(Date.now())}_${randomBytes(4).toString("hex")}`;
+}
+async function readManualActionPageMetadata(page) {
+    let pageUrl;
+    let pageTitle;
+    try {
+        pageUrl = page.url();
+    }
+    catch {
+    }
+    try {
+        pageTitle = await page.title();
+    }
+    catch {
+    }
+    return {
+        ...(pageUrl ? { pageUrl } : {}),
+        ...(pageTitle ? { pageTitle } : {}),
+    };
+}
+function registerManagedNekoManualActionTarget(args) {
+    const nekoDescriptor = resolveManagedNekoDescriptorFromEnv(args.env);
+    if (!nekoDescriptor) {
+        process.stderr.write(`[browser-handoff] managed n.eko surface env is incomplete; streaming-companion target not registered for interaction ${args.interactionId}.\n`);
+        return Promise.resolve(false);
+    }
+    return args.registration.register({
+        backend: "neko",
+        runId: args.registration.runId,
+        interactionId: args.interactionId,
+        descriptor: {
+            ...nekoDescriptor,
+            interaction_id: args.interactionId,
+            ...(args.metadata.pageUrl ? { start_url: args.metadata.pageUrl } : {}),
+        },
+        ...(args.metadata.pageUrl ? { pageUrl: args.metadata.pageUrl } : {}),
+        ...(args.metadata.pageTitle ? { pageTitle: args.metadata.pageTitle } : {}),
+        ...(args.reason ? { reason: args.reason } : {}),
+    });
+}
+async function resolveCdpWsUrlForManualAction(args) {
+    try {
+        return await args.resolveWsUrl(args.page, args.endpoint);
+    }
+    catch (err) {
+        const message = err instanceof Error ? err.message : String(err);
+        process.stderr.write(`[browser-handoff] could not resolve CDP page-target wsUrl for interaction ${args.interactionId}: ${message}; continuing without streaming.\n`);
+        return;
+    }
+}
+function registerCdpManualActionTarget(args) {
+    return args.registration.register({
+        runId: args.registration.runId,
+        interactionId: args.interactionId,
+        wsUrl: args.wsUrl,
+        ...(args.metadata.pageUrl ? { pageUrl: args.metadata.pageUrl } : {}),
+        ...(args.metadata.pageTitle ? { pageTitle: args.metadata.pageTitle } : {}),
+        ...(args.reason ? { reason: args.reason } : {}),
+    });
+}
+export async function prepareManualAction(args) {
+    const env = args.env ?? process.env;
+    const resolveStreamingRegistration = args.resolveStreamingRegistration ?? resolveStreamingRegistrationFromEnv;
+    const resolveWsUrl = args.resolveWsUrl ?? resolveWsUrlForExactPage;
+    const interactionId = generateInteractionId();
+    const registration = await resolveStreamingRegistration(env);
+    if (!registration) {
+        return { interactionId, registered: false };
+    }
+    const metadata = await readManualActionPageMetadata(args.page);
+    if (isManagedNekoRequired(env)) {
+        const ok = await registerManagedNekoManualActionTarget({
+            env,
+            interactionId,
+            metadata,
+            registration,
+            ...(args.reason ? { reason: args.reason } : {}),
+        });
+        return { interactionId, registered: ok };
+    }
+    const endpoint = resolveCdpEndpointFromEnv(env);
+    if (!endpoint) {
+        process.stderr.write(`[browser-handoff] ${BROWSER_CDP_HOST_ENV}/${BROWSER_CDP_PORT_ENV} not set; streaming-companion target not registered for interaction ${interactionId}.\n`);
+        return { interactionId, registered: false };
+    }
+    const wsUrl = await resolveCdpWsUrlForManualAction({
+        endpoint,
+        interactionId,
+        page: args.page,
+        resolveWsUrl,
+    });
+    if (!wsUrl) {
+        return { interactionId, registered: false };
+    }
+    const ok = await registerCdpManualActionTarget({
+        interactionId,
+        metadata,
+        registration,
+        wsUrl,
+        ...(args.reason ? { reason: args.reason } : {}),
+    });
+    if (!ok) {
+        return { interactionId, registered: false };
+    }
+    return { interactionId, registered: true };
+}
+function captureManualActionFixture(args) {
+    if (!args.capture) {
+        return;
+    }
+    try {
+        const capture = args.capture.captureDom(args.page, `manual-action-${args.reason ?? "manual_action"}-${args.interactionId}`);
+        capture.catch(() => undefined);
+    }
+    catch {
+    }
+}
+export async function manualAction(args, sendInteraction) {
+    const { interactionId } = await prepareManualAction({
+        page: args.page,
+        ...(args.reason ? { reason: args.reason } : {}),
+        ...(args.env ? { env: args.env } : {}),
+        ...(args.resolveStreamingRegistration ? { resolveStreamingRegistration: args.resolveStreamingRegistration } : {}),
+        ...(args.resolveWsUrl ? { resolveWsUrl: args.resolveWsUrl } : {}),
+    });
+    captureManualActionFixture({
+        ...(args.capture ? { capture: args.capture } : {}),
+        interactionId,
+        page: args.page,
+        ...(args.reason ? { reason: args.reason } : {}),
+    });
+    return await sendInteraction({
+        kind: "manual_action",
+        request_id: interactionId,
+        message: args.message,
+        ...(args.schema ? { schema: args.schema } : {}),
+        ...(args.timeoutSeconds === undefined ? {} : { timeout_seconds: args.timeoutSeconds }),
+    });
+}
+export { BROWSER_CDP_HOST_ENV, BROWSER_CDP_PORT_ENV, BROWSER_SURFACE_ID_ENV, BROWSER_SURFACE_LEASE_ID_ENV, BROWSER_SURFACE_PROFILE_KEY_ENV, BROWSER_SURFACE_REQUIRED_ENV, BROWSER_SURFACE_STREAM_BASE_URL_ENV, };

package/dist/polyfill-connectors/src/collector-protocol.d.ts

@@ -0,0 +1,2 @@
+export declare const COLLECTOR_PROTOCOL_VERSION = "1";
+export declare const COLLECTOR_PROTOCOL_HEADER = "X-PDPP-Collector-Protocol";

package/dist/polyfill-connectors/src/collector-protocol.js

@@ -0,0 +1,2 @@
+export const COLLECTOR_PROTOCOL_VERSION = "1";
+export const COLLECTOR_PROTOCOL_HEADER = "X-PDPP-Collector-Protocol";