@pdpp/local-collector 0.0.0

package/dist/polyfill-connectors/src/fixture-capture.js

@@ -0,0 +1,237 @@
+import { appendFileSync, mkdirSync, rmSync, writeFileSync } from "node:fs";
+import { dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
+const PACKAGE_ROOT = dirname(dirname(fileURLToPath(import.meta.url)));
+const ARIA_SNAPSHOT_TIMEOUT_MS = 2000;
+const LOCATOR_PROBE_TIMEOUT_MS = 1000;
+const LOCATOR_PROBE_ARIA_DEPTH = 2;
+const safeLabel = (s) => String(s)
+    .replace(/[^A-Za-z0-9_.-]/g, "_")
+    .slice(0, 120);
+function requireProbeMethod(page, key) {
+    const method = page[key];
+    if (typeof method !== "function") {
+        throw new Error(`locator probe page is missing ${String(key)}`);
+    }
+    return method;
+}
+function locatorForProbe(page, probe) {
+    switch (probe.kind) {
+        case "css":
+            return page.locator(probe.selector);
+        case "label":
+            return requireProbeMethod(page, "getByLabel")(probe.text, probe.exact === undefined ? undefined : { exact: probe.exact });
+        case "placeholder":
+            return requireProbeMethod(page, "getByPlaceholder")(probe.text, probe.exact === undefined ? undefined : { exact: probe.exact });
+        case "role": {
+            const name = probe.namePattern ? new RegExp(probe.namePattern, probe.nameFlags ?? "i") : probe.name;
+            return requireProbeMethod(page, "getByRole")(probe.role, {
+                ...(probe.exact === undefined ? {} : { exact: probe.exact }),
+                ...(name === undefined ? {} : { name }),
+            });
+        }
+        case "text":
+            return requireProbeMethod(page, "getByText")(probe.text, probe.exact === undefined ? undefined : { exact: probe.exact });
+        default:
+            throw new Error(`unsupported locator probe kind: ${probe.kind ?? "unknown"}`);
+    }
+}
+function probeForReport(probe) {
+    const { description: _description, id: _id, kind: _kind, ...rest } = probe;
+    return rest;
+}
+async function captureDomHtml(page, baseDir, label, safe) {
+    try {
+        const html = await page.content();
+        writeFileSync(join(baseDir, "dom", `${safe}.html`), html);
+    }
+    catch (err) {
+        const message = err instanceof Error ? err.message : String(err);
+        process.stderr.write(`[capture] dom write failed for ${label}: ${message}\n`);
+    }
+}
+async function capturePageMetadata(page, baseDir, label, safe) {
+    try {
+        const title = await page.title().catch(() => "");
+        writeFileSync(join(baseDir, "pages", `${safe}.json`), JSON.stringify({
+            captured_at: new Date().toISOString(),
+            label,
+            title,
+            url: page.url(),
+        }, null, 2));
+    }
+    catch (err) {
+        const message = err instanceof Error ? err.message : String(err);
+        process.stderr.write(`[capture] page metadata write failed for ${label}: ${message}\n`);
+    }
+}
+async function captureAriaSnapshot(page, baseDir, label, safe) {
+    try {
+        const ariaSnapshot = await page.ariaSnapshot({
+            depth: Number(process.env.PDPP_CAPTURE_ARIA_DEPTH ?? 8),
+            mode: "ai",
+            timeout: ARIA_SNAPSHOT_TIMEOUT_MS,
+        });
+        writeFileSync(join(baseDir, "aria", `${safe}.aria.yml`), ariaSnapshot);
+    }
+    catch (err) {
+        const message = err instanceof Error ? err.message : String(err);
+        process.stderr.write(`[capture] aria snapshot failed for ${label}: ${message}\n`);
+    }
+}
+async function captureScreenshot(page, baseDir, label, safe) {
+    try {
+        const screenshot = await page.screenshot({ fullPage: false, type: "png" });
+        writeFileSync(join(baseDir, "screenshots", `${safe}.png`), screenshot);
+    }
+    catch (err) {
+        const message = err instanceof Error ? err.message : String(err);
+        process.stderr.write(`[capture] screenshot write failed for ${label}: ${message}\n`);
+    }
+}
+async function runLocatorProbe(page, probe) {
+    const result = {
+        id: probe.id,
+        kind: probe.kind,
+        probe: probeForReport(probe),
+    };
+    if (probe.description !== undefined) {
+        result.description = probe.description;
+    }
+    try {
+        const locator = locatorForProbe(page, probe);
+        result.count = await locator.count();
+        if (result.count > 0) {
+            const first = locator.first();
+            result.visible = await first.isVisible();
+            result.enabled = await first.isEnabled({ timeout: LOCATOR_PROBE_TIMEOUT_MS }).catch(() => false);
+            const ariaSnapshot = await first
+                .ariaSnapshot({
+                depth: LOCATOR_PROBE_ARIA_DEPTH,
+                mode: "ai",
+                timeout: LOCATOR_PROBE_TIMEOUT_MS,
+            })
+                .catch(() => undefined);
+            if (ariaSnapshot !== undefined) {
+                result.ariaSnapshot = ariaSnapshot;
+            }
+        }
+    }
+    catch (err) {
+        result.error = err instanceof Error ? err.message : String(err);
+    }
+    return result;
+}
+async function writeLocatorProbeReport(page, baseDir, label, safe, results) {
+    try {
+        writeFileSync(join(baseDir, "locators", `${safe}.json`), JSON.stringify({
+            captured_at: new Date().toISOString(),
+            label,
+            probes: results,
+            title: await page.title().catch(() => ""),
+            url: page.url(),
+        }, null, 2));
+    }
+    catch (err) {
+        const message = err instanceof Error ? err.message : String(err);
+        process.stderr.write(`[capture] locator probe write failed for ${label}: ${message}\n`);
+    }
+}
+export function createCaptureSession(connectorName) {
+    const alwaysRetain = process.env.PDPP_CAPTURE_FIXTURES === "1";
+    const onFailureOnly = process.env.PDPP_CAPTURE_ON_FAILURE === "1";
+    if (!(alwaysRetain || onFailureOnly)) {
+        return null;
+    }
+    const keepOnSuccess = alwaysRetain;
+    const runId = new Date().toISOString().replace(/[:.]/g, "-");
+    const baseDir = join(PACKAGE_ROOT, "fixtures", connectorName, "raw", runId);
+    try {
+        mkdirSync(join(baseDir, "records"), { recursive: true });
+        mkdirSync(join(baseDir, "aria"), { recursive: true });
+        mkdirSync(join(baseDir, "dom"), { recursive: true });
+        mkdirSync(join(baseDir, "locators"), { recursive: true });
+        mkdirSync(join(baseDir, "pages"), { recursive: true });
+        mkdirSync(join(baseDir, "screenshots"), { recursive: true });
+        mkdirSync(join(baseDir, "traces"), { recursive: true });
+        mkdirSync(join(baseDir, "http"), { recursive: true });
+    }
+    catch (err) {
+        const message = err instanceof Error ? err.message : String(err);
+        process.stderr.write(`[capture] mkdir failed: ${message}\n`);
+        return null;
+    }
+    let httpSeq = 0;
+    let traceCheckpointHook = null;
+    let succeeded = false;
+    let finalized = false;
+    return {
+        runId,
+        baseDir,
+        keepOnSuccess,
+        setTraceCheckpointHook(hook) {
+            traceCheckpointHook = hook;
+        },
+        markSucceeded() {
+            succeeded = true;
+        },
+        finalize() {
+            if (finalized) {
+                return;
+            }
+            finalized = true;
+            if (keepOnSuccess || !succeeded) {
+                return;
+            }
+            try {
+                rmSync(baseDir, { force: true, recursive: true });
+                process.stderr.write(`[capture] run succeeded; raw capture deleted (${baseDir})\n`);
+            }
+            catch (err) {
+                const message = err instanceof Error ? err.message : String(err);
+                process.stderr.write(`[capture] cleanup failed for ${baseDir}: ${message}\n`);
+            }
+        },
+        recordRecord(msg) {
+            try {
+                const file = join(baseDir, "records", `${safeLabel(msg.stream)}.jsonl`);
+                appendFileSync(file, `${JSON.stringify(msg.data)}\n`);
+            }
+            catch (err) {
+                const message = err instanceof Error ? err.message : String(err);
+                process.stderr.write(`[capture] record write failed: ${message}\n`);
+            }
+        },
+        async captureDom(page, label) {
+            const safe = safeLabel(label);
+            await captureDomHtml(page, baseDir, label, safe);
+            await capturePageMetadata(page, baseDir, label, safe);
+            await captureAriaSnapshot(page, baseDir, label, safe);
+            await captureScreenshot(page, baseDir, label, safe);
+            if (traceCheckpointHook) {
+                await traceCheckpointHook(label).catch((err) => {
+                    const message = err instanceof Error ? err.message : String(err);
+                    process.stderr.write(`[capture] trace checkpoint failed for ${label}: ${message}\n`);
+                });
+            }
+        },
+        async captureLocatorProbe(page, label, probes) {
+            const safe = safeLabel(label);
+            const results = await Promise.all(probes.map((probe) => runLocatorProbe(page, probe)));
+            await writeLocatorProbeReport(page, baseDir, label, safe, results);
+        },
+        captureHttp(label, body, meta = {}) {
+            try {
+                httpSeq += 1;
+                const idx = String(httpSeq).padStart(4, "0");
+                const file = join(baseDir, "http", `${idx}-${safeLabel(label)}.json`);
+                const payload = { label, meta, body };
+                writeFileSync(file, JSON.stringify(payload, null, 2));
+            }
+            catch (err) {
+                const message = err instanceof Error ? err.message : String(err);
+                process.stderr.write(`[capture] http write failed for ${label}: ${message}\n`);
+            }
+        },
+    };
+}

package/dist/polyfill-connectors/src/is-main-module.d.ts

@@ -0,0 +1 @@
+export declare function isMainModule(importMetaUrl: string): boolean;

package/dist/polyfill-connectors/src/is-main-module.js

@@ -0,0 +1,17 @@
+import { realpathSync } from "node:fs";
+import { fileURLToPath, pathToFileURL } from "node:url";
+export function isMainModule(importMetaUrl) {
+    const entry = process.argv[1];
+    if (!entry) {
+        return false;
+    }
+    if (importMetaUrl === pathToFileURL(entry).href) {
+        return true;
+    }
+    try {
+        return realpathSync(fileURLToPath(importMetaUrl)) === realpathSync(entry);
+    }
+    catch {
+        return false;
+    }
+}

package/dist/polyfill-connectors/src/local-device-client.d.ts

@@ -0,0 +1,126 @@
+import type { LocalDeviceRecordEnvelope } from "./local-device-envelope.js";
+export declare const LOCAL_DEVICE_ENDPOINTS: {
+    readonly exchangeEnrollment: "/_ref/device-exporters/enroll";
+    readonly heartbeat: (deviceId: string) => string;
+    readonly ingestBatch: (deviceId: string) => string;
+    readonly localCollectorGap: (deviceId: string, sourceInstanceId: string) => string;
+    readonly localCollectorGapRecovered: (deviceId: string, sourceInstanceId: string) => string;
+    readonly sourceInstanceState: (deviceId: string, sourceInstanceId: string) => string;
+};
+export interface LocalDeviceClientOptions {
+    baseUrl: string;
+    deviceId?: string;
+    deviceToken?: string;
+    fetchImpl?: typeof fetch;
+}
+export interface EnrollmentExchangeRequest {
+    device_label?: string;
+    enrollment_code: string;
+}
+export interface EnrollmentExchangeResponse {
+    connector_id: string;
+    device_id: string;
+    device_token: string;
+    local_binding_name: string;
+    source_instance_id: string;
+}
+export interface HeartbeatOutboxDiagnostics {
+    backlog_open?: number;
+    dead_letter: number;
+    leased: number;
+    oldest_pending_at?: string | null;
+    pending: number;
+    retrying: number;
+    stale_leases: number;
+    succeeded: number;
+    total: number;
+}
+export interface HeartbeatRequest {
+    connector_id: string;
+    outbox?: HeartbeatOutboxDiagnostics;
+    records_pending?: number;
+    source_instance_id: string;
+    status: "starting" | "healthy" | "retrying" | "blocked" | "stopped";
+}
+export interface IngestBatchRequest {
+    batch_id: string;
+    batch_seq: number;
+    body_hash: string;
+    connector_id: string;
+    device_id: string;
+    records: Pick<LocalDeviceRecordEnvelope, "data" | "emitted_at" | "record_key" | "stream">[];
+    source_instance_id: string;
+}
+export interface GetSourceInstanceStateRequest {
+    sourceInstanceId: string;
+}
+export interface PutSourceInstanceStateRequest {
+    sourceInstanceId: string;
+    state: Record<string, unknown>;
+}
+export interface SourceInstanceStateResponse {
+    device_id: string;
+    object: "device_source_instance_state";
+    source_instance_id: string;
+    state: Record<string, unknown>;
+    updated_at: string | null;
+}
+export interface AckLocalCollectorGapRequest {
+    connector_id: string;
+    details?: string;
+    first_seen_at: string;
+    first_seen_run_id?: string;
+    last_run_id?: string;
+    next_attempt_backoff_ms: number;
+    reason: "policy_budget" | "connector_child_failure";
+    retryable: boolean;
+    source_instance_id: string;
+    stream?: string;
+    stream_boundary?: string;
+}
+export interface AckLocalCollectorGapResponse {
+    attempt_count: number;
+    connector_id: string;
+    connector_instance_id: string;
+    device_id: string;
+    first_seen_at: string | null;
+    first_seen_run_id: string | null;
+    gap_id: string;
+    last_run_id: string | null;
+    object: "device_local_collector_gap";
+    reason: "policy_budget" | "connector_child_failure";
+    retryable: boolean;
+    source_instance_id: string;
+    status: string;
+    stream: string;
+    updated_at: string | null;
+}
+export interface RecoverLocalCollectorGapRequest {
+    connector_id: string;
+    reason: "policy_budget" | "connector_child_failure";
+    recovered_run_id?: string;
+    source_instance_id: string;
+    stream?: string;
+    stream_boundary?: string;
+}
+export declare class LocalDeviceHttpError extends Error {
+    readonly body: string;
+    readonly status: number;
+    readonly code: string | null;
+    constructor(status: number, body: string);
+}
+export declare class LocalDeviceClient {
+    #private;
+    constructor(options: LocalDeviceClientOptions);
+    exchangeEnrollment(request: EnrollmentExchangeRequest): Promise<EnrollmentExchangeResponse>;
+    heartbeat(request: HeartbeatRequest): Promise<{
+        ok: true;
+    }>;
+    ingestBatch(request: IngestBatchRequest): Promise<{
+        ok: true;
+    }>;
+    getSourceInstanceState(request: GetSourceInstanceStateRequest): Promise<SourceInstanceStateResponse>;
+    putSourceInstanceState(request: PutSourceInstanceStateRequest): Promise<SourceInstanceStateResponse>;
+    ackLocalCollectorGap(request: AckLocalCollectorGapRequest): Promise<AckLocalCollectorGapResponse>;
+    recoverLocalCollectorGap(request: RecoverLocalCollectorGapRequest): Promise<AckLocalCollectorGapResponse>;
+}

package/dist/polyfill-connectors/src/local-device-client.js

@@ -0,0 +1,132 @@
+import { COLLECTOR_PROTOCOL_HEADER, COLLECTOR_PROTOCOL_VERSION } from "./collector-protocol.js";
+export const LOCAL_DEVICE_ENDPOINTS = {
+    exchangeEnrollment: "/_ref/device-exporters/enroll",
+    heartbeat: (deviceId) => `/_ref/device-exporters/${encodeURIComponent(deviceId)}/heartbeat`,
+    ingestBatch: (deviceId) => `/_ref/device-exporters/${encodeURIComponent(deviceId)}/ingest-batches`,
+    localCollectorGap: (deviceId, sourceInstanceId) => `/_ref/device-exporters/${encodeURIComponent(deviceId)}/source-instances/${encodeURIComponent(sourceInstanceId)}/local-collector-gaps`,
+    localCollectorGapRecovered: (deviceId, sourceInstanceId) => `/_ref/device-exporters/${encodeURIComponent(deviceId)}/source-instances/${encodeURIComponent(sourceInstanceId)}/local-collector-gaps/recovered`,
+    sourceInstanceState: (deviceId, sourceInstanceId) => `/_ref/device-exporters/${encodeURIComponent(deviceId)}/source-instances/${encodeURIComponent(sourceInstanceId)}/state`,
+};
+export class LocalDeviceHttpError extends Error {
+    body;
+    status;
+    code;
+    constructor(status, body) {
+        const parsed = parseLocalDeviceErrorEnvelope(body);
+        const detail = parsed?.code ? ` ${parsed.code}` : "";
+        super(`local device request failed: ${status}${detail}`);
+        this.name = "LocalDeviceHttpError";
+        this.status = status;
+        this.body = body;
+        this.code = parsed?.code ?? null;
+    }
+}
+function parseLocalDeviceErrorEnvelope(body) {
+    if (!body) {
+        return null;
+    }
+    try {
+        const parsed = JSON.parse(body);
+        if (parsed && typeof parsed === "object" && parsed.error && typeof parsed.error.code === "string") {
+            return { code: parsed.error.code };
+        }
+    }
+    catch {
+    }
+    return null;
+}
+export class LocalDeviceClient {
+    #baseUrl;
+    #deviceId;
+    #deviceToken;
+    #fetch;
+    constructor(options) {
+        this.#baseUrl = new URL(options.baseUrl);
+        this.#deviceId = options.deviceId;
+        this.#deviceToken = options.deviceToken;
+        this.#fetch = options.fetchImpl ?? fetch;
+    }
+    exchangeEnrollment(request) {
+        return this.#request(LOCAL_DEVICE_ENDPOINTS.exchangeEnrollment, {
+            authenticate: false,
+            body: request,
+            method: "POST",
+        });
+    }
+    heartbeat(request) {
+        return this.#request(LOCAL_DEVICE_ENDPOINTS.heartbeat(this.#requireDeviceId()), {
+            authenticate: true,
+            body: request,
+            method: "POST",
+        });
+    }
+    ingestBatch(request) {
+        return this.#request(LOCAL_DEVICE_ENDPOINTS.ingestBatch(this.#requireDeviceId()), {
+            authenticate: true,
+            body: request,
+            method: "POST",
+        });
+    }
+    getSourceInstanceState(request) {
+        const path = LOCAL_DEVICE_ENDPOINTS.sourceInstanceState(this.#requireDeviceId(), request.sourceInstanceId);
+        return this.#request(path, { authenticate: true, method: "GET" });
+    }
+    putSourceInstanceState(request) {
+        const path = LOCAL_DEVICE_ENDPOINTS.sourceInstanceState(this.#requireDeviceId(), request.sourceInstanceId);
+        return this.#request(path, {
+            authenticate: true,
+            body: { state: request.state },
+            method: "PUT",
+        });
+    }
+    ackLocalCollectorGap(request) {
+        const path = LOCAL_DEVICE_ENDPOINTS.localCollectorGap(this.#requireDeviceId(), request.source_instance_id);
+        return this.#request(path, {
+            authenticate: true,
+            body: request,
+            method: "POST",
+        });
+    }
+    recoverLocalCollectorGap(request) {
+        const path = LOCAL_DEVICE_ENDPOINTS.localCollectorGapRecovered(this.#requireDeviceId(), request.source_instance_id);
+        return this.#request(path, {
+            authenticate: true,
+            body: request,
+            method: "POST",
+        });
+    }
+    #requireDeviceId() {
+        if (!this.#deviceId) {
+            throw new Error("device id required for authenticated local device request");
+        }
+        return this.#deviceId;
+    }
+    async #request(path, options) {
+        const headers = {
+            accept: "application/json",
+            [COLLECTOR_PROTOCOL_HEADER]: COLLECTOR_PROTOCOL_VERSION,
+        };
+        if (options.body !== undefined) {
+            headers["content-type"] = "application/json";
+        }
+        if (options.authenticate) {
+            if (!this.#deviceToken) {
+                throw new Error("device token required for authenticated local device request");
+            }
+            headers.authorization = `Bearer ${this.#deviceToken}`;
+        }
+        const init = { headers, method: options.method };
+        if (options.body !== undefined) {
+            init.body = JSON.stringify(options.body);
+        }
+        const response = await this.#fetch(new URL(path, this.#baseUrl), init);
+        const text = await response.text();
+        if (!response.ok) {
+            throw new LocalDeviceHttpError(response.status, text);
+        }
+        if (!text) {
+            return { ok: true };
+        }
+        return JSON.parse(text);
+    }
+}

package/dist/polyfill-connectors/src/local-device-envelope.d.ts

@@ -0,0 +1,26 @@
+import type { EmittedMessage, RecordData } from "./connector-runtime-protocol.js";
+export interface LocalDeviceRecordEnvelope {
+    batch_id: string;
+    batch_seq: number;
+    body_hash: string;
+    connector_id: string;
+    data: RecordData;
+    device_id: string;
+    emitted_at: string;
+    record_key: string;
+    source_instance_id: string;
+    stream: string;
+}
+export interface BuildLocalDeviceRecordEnvelopeInput {
+    batchId: string;
+    batchSeq: number;
+    connectorId: string;
+    deviceId: string;
+    record: Extract<EmittedMessage, {
+        type: "RECORD";
+    }>;
+    sourceInstanceId: string;
+}
+export declare function canonicalJson(value: unknown): string;
+export declare function hashCanonicalJson(value: unknown): string;
+export declare function buildLocalDeviceRecordEnvelope(input: BuildLocalDeviceRecordEnvelopeInput): LocalDeviceRecordEnvelope;

package/dist/polyfill-connectors/src/local-device-envelope.js

@@ -0,0 +1,43 @@
+import { createHash } from "node:crypto";
+export function canonicalJson(value) {
+    return JSON.stringify(toCanonicalValue(value));
+}
+export function hashCanonicalJson(value) {
+    return createHash("sha256").update(canonicalJson(value)).digest("hex");
+}
+export function buildLocalDeviceRecordEnvelope(input) {
+    const body = {
+        connector_id: input.connectorId,
+        data: toNormalizedRecordData(input.record.data),
+        emitted_at: input.record.emitted_at,
+        record_key: String(input.record.key),
+        stream: input.record.stream,
+    };
+    return {
+        batch_id: input.batchId,
+        batch_seq: input.batchSeq,
+        body_hash: hashCanonicalJson(body),
+        device_id: input.deviceId,
+        source_instance_id: input.sourceInstanceId,
+        ...body,
+    };
+}
+function toCanonicalValue(value) {
+    if (value === null || typeof value !== "object") {
+        return value;
+    }
+    if (Array.isArray(value)) {
+        return value.map((item) => toCanonicalValue(item));
+    }
+    const out = {};
+    for (const key of Object.keys(value).sort()) {
+        const item = value[key];
+        if (item !== undefined) {
+            out[key] = toCanonicalValue(item);
+        }
+    }
+    return out;
+}
+function toNormalizedRecordData(data) {
+    return toCanonicalValue(data);
+}