@kontourai/lookout 0.1.0

package/dist/src/proposal-diff.js

@@ -0,0 +1,208 @@
+import { canonicalValueKey, } from "./canonical-value.js";
+import { compareStructural, diffKeyedMultiset } from "./structural-diff.js";
+function callbackError(label, cause) {
+    return { kind: "callback-threw", message: `${label} callback threw`, cause };
+}
+function invoke(label, callback) {
+    try {
+        return { ok: true, value: callback() };
+    }
+    catch (cause) {
+        return { ok: false, error: callbackError(label, cause) };
+    }
+}
+function identity(label, callback) {
+    const called = invoke(label, callback);
+    if (!called.ok)
+        return called;
+    if (typeof called.value === "string")
+        return { ok: true, value: called.value };
+    return called.value.ok ? { ok: true, value: called.value.key } : called.value;
+}
+export function extractionProposalIdentity(proposal) {
+    try {
+        const pathIndices = Object.prototype.hasOwnProperty.call(proposal, "pathIndices")
+            ? { present: true, value: proposal.pathIndices }
+            : { present: false };
+        const encoded = canonicalValueKey({
+            fieldPath: proposal.fieldPath,
+            pathIndices,
+            locator: proposal.provenance.locator,
+        });
+        return encoded.ok ? { ok: true, key: encoded.key } : encoded;
+    }
+    catch (cause) {
+        return {
+            ok: false,
+            error: { kind: "unsupported-value", message: "Extraction proposal identity could not be inspected", path: "$", cause },
+        };
+    }
+}
+function evidence(observation, entityKey, fieldKey, proposal) {
+    return {
+        sourceId: observation.sourceId,
+        snapshotRef: observation.snapshotRef,
+        observedAt: observation.observedAt,
+        entityKey,
+        fieldKey,
+        value: proposal.candidateValue,
+        confidence: proposal.confidence,
+        provenance: proposal.provenance,
+        extractor: proposal.extractor,
+        fieldPath: proposal.fieldPath,
+        ...(Object.prototype.hasOwnProperty.call(proposal, "pathIndices")
+            ? { pathIndices: proposal.pathIndices }
+            : {}),
+    };
+}
+function changeKind(prior, current) {
+    if (prior === undefined && current !== undefined)
+        return { ok: true, value: "value-populated" };
+    if (Array.isArray(prior) && Array.isArray(current)) {
+        const facts = diffKeyedMultiset(prior, current, { identity: canonicalValueKey });
+        if (!facts.ok)
+            return facts;
+        const added = facts.value.additions.length > 0;
+        const removed = facts.value.removals.length > 0;
+        if (added && !removed)
+            return { ok: true, value: "items-added" };
+        if (removed && !added)
+            return { ok: true, value: "items-removed" };
+        return { ok: true, value: "value-replaced" };
+    }
+    const sameCategory = (prior === null ? "null" : typeof prior) === (current === null ? "null" : typeof current);
+    return { ok: true, value: sameCategory ? "value-updated" : "value-replaced" };
+}
+export function diffProposalSets(input) {
+    if (input.prior.sourceId !== input.current.sourceId) {
+        return {
+            ok: false,
+            error: { kind: "unsupported-value", message: "Proposal observations must have the same sourceId", path: "$.sourceId" },
+        };
+    }
+    const priorEntities = invoke("selectEntities", () => input.selectEntities(input.prior));
+    if (!priorEntities.ok)
+        return priorEntities;
+    const currentEntities = invoke("selectEntities", () => input.selectEntities(input.current));
+    if (!currentEntities.ok)
+        return currentEntities;
+    const entities = diffKeyedMultiset(priorEntities.value, currentEntities.value, {
+        identity: (entity) => input.entityIdentity(entity),
+    });
+    if (!entities.ok)
+        return entities;
+    const events = [];
+    const retainedProposalOccurrences = [];
+    const addedProposalOccurrences = [];
+    const removedProposalOccurrences = [];
+    const provenanceChanges = [];
+    const removedEntities = [];
+    for (const pair of entities.value.retained) {
+        const entityKeyResult = identity("entityIdentity", () => input.entityIdentity(pair.prior));
+        if (!entityKeyResult.ok)
+            return entityKeyResult;
+        const entityKey = entityKeyResult.value;
+        const priorProposals = invoke("proposalsFor", () => input.proposalsFor(pair.prior));
+        if (!priorProposals.ok)
+            return priorProposals;
+        const currentProposals = invoke("proposalsFor", () => input.proposalsFor(pair.current));
+        if (!currentProposals.ok)
+            return currentProposals;
+        const occurrences = diffKeyedMultiset(priorProposals.value, currentProposals.value, {
+            identity: extractionProposalIdentity,
+        });
+        if (!occurrences.ok)
+            return occurrences;
+        retainedProposalOccurrences.push(...occurrences.value.retained);
+        addedProposalOccurrences.push(...occurrences.value.additions);
+        removedProposalOccurrences.push(...occurrences.value.removals);
+        const priorFields = priorProposals.value.map((proposal) => ({ entity: pair.prior, proposal }));
+        const currentFields = currentProposals.value.map((proposal) => ({ entity: pair.current, proposal }));
+        const fields = diffKeyedMultiset(priorFields, currentFields, {
+            identity: ({ entity, proposal }) => input.fieldIdentity(entity, proposal),
+        });
+        if (!fields.ok)
+            return fields;
+        for (const field of fields.value.retained) {
+            const fieldKeyResult = identity("fieldIdentity", () => input.fieldIdentity(field.prior.entity, field.prior.proposal));
+            if (!fieldKeyResult.ok)
+                return fieldKeyResult;
+            const fieldKey = fieldKeyResult.value;
+            const comparison = compareStructural(field.prior.proposal, field.current.proposal, {
+                value: (proposal) => proposal.candidateValue,
+                provenance: (proposal) => proposal.provenance,
+            });
+            if (!comparison.ok)
+                return comparison;
+            const priorEvidence = evidence(input.prior, entityKey, fieldKey, field.prior.proposal);
+            const currentEvidence = evidence(input.current, entityKey, fieldKey, field.current.proposal);
+            if (comparison.value.provenanceChanged) {
+                provenanceChanges.push({ entityKey, fieldKey, prior: priorEvidence, current: currentEvidence });
+            }
+            if (comparison.value.valueChanged) {
+                const kind = changeKind(field.prior.proposal.candidateValue, field.current.proposal.candidateValue);
+                if (!kind.ok)
+                    return kind;
+                events.push({ kind: "field-changed", entityKey, fieldKey, changeKind: kind.value, prior: priorEvidence, current: currentEvidence });
+            }
+        }
+        for (const field of fields.value.additions) {
+            const fieldKey = identity("fieldIdentity", () => input.fieldIdentity(field.entity, field.proposal));
+            if (!fieldKey.ok)
+                return fieldKey;
+            events.push({
+                kind: "field-changed",
+                entityKey,
+                fieldKey: fieldKey.value,
+                changeKind: "value-populated",
+                current: evidence(input.current, entityKey, fieldKey.value, field.proposal),
+            });
+        }
+        for (const field of fields.value.removals) {
+            const fieldKey = identity("fieldIdentity", () => input.fieldIdentity(field.entity, field.proposal));
+            if (!fieldKey.ok)
+                return fieldKey;
+            events.push({
+                kind: "field-changed",
+                entityKey,
+                fieldKey: fieldKey.value,
+                changeKind: Array.isArray(field.proposal.candidateValue) ? "items-removed" : "value-replaced",
+                prior: evidence(input.prior, entityKey, fieldKey.value, field.proposal),
+            });
+        }
+    }
+    for (const entity of entities.value.additions) {
+        const entityKeyResult = identity("entityIdentity", () => input.entityIdentity(entity));
+        if (!entityKeyResult.ok)
+            return entityKeyResult;
+        const proposals = invoke("proposalsFor", () => input.proposalsFor(entity));
+        if (!proposals.ok)
+            return proposals;
+        const current = [];
+        for (const proposal of proposals.value) {
+            const fieldKey = identity("fieldIdentity", () => input.fieldIdentity(entity, proposal));
+            if (!fieldKey.ok)
+                return fieldKey;
+            current.push(evidence(input.current, entityKeyResult.value, fieldKey.value, proposal));
+            addedProposalOccurrences.push(proposal);
+        }
+        events.push({ kind: "new-entity-appeared", entityKey: entityKeyResult.value, current });
+    }
+    for (const entity of entities.value.removals) {
+        const entityKeyResult = identity("entityIdentity", () => input.entityIdentity(entity));
+        if (!entityKeyResult.ok)
+            return entityKeyResult;
+        removedEntities.push(entityKeyResult.value);
+        const proposals = invoke("proposalsFor", () => input.proposalsFor(entity));
+        if (!proposals.ok)
+            return proposals;
+        removedProposalOccurrences.push(...proposals.value);
+    }
+    return {
+        ok: true,
+        value: {
+            events,
+            facts: { retainedProposalOccurrences, addedProposalOccurrences, removedProposalOccurrences, provenanceChanges, removedEntities },
+        },
+    };
+}

package/dist/src/provider-resolution.d.ts

@@ -0,0 +1,4 @@
+import { resolve } from "@kontourai/datum";
+/** L2 capability seam. L1 accepts this capability but never invokes it. */
+export type ProviderResolver = typeof resolve;
+export declare const defaultProviderResolver: ProviderResolver;

package/dist/src/provider-resolution.js

@@ -0,0 +1,2 @@
+import { resolve } from "@kontourai/datum";
+export const defaultProviderResolver = resolve;

package/dist/src/registry.d.ts

@@ -0,0 +1,29 @@
+import type { TargetFieldSchema } from "@kontourai/traverse";
+export type LookoutSourceKind = "web-page" | "api-record";
+export type RenderPolicy = "never" | "on-shell-warning" | "always";
+export interface LookoutSource {
+    id: string;
+    kind: LookoutSourceKind;
+    url: string;
+    targetSchema: TargetFieldSchema[];
+    cadenceHint: string;
+    renderPolicy: RenderPolicy;
+}
+export interface LookoutRegistryDocument {
+    version: 1;
+    sources: LookoutSource[];
+}
+export declare class RegistryValidationError extends Error {
+    readonly issues: string[];
+    constructor(issues: string[]);
+}
+export declare class LookoutRegistry {
+    #private;
+    readonly version: 1;
+    readonly sources: readonly LookoutSource[];
+    constructor(sources: LookoutSource[]);
+    list(): readonly LookoutSource[];
+    get(id: string): LookoutSource | undefined;
+}
+export declare function loadRegistry(registryPath?: string): Promise<LookoutRegistry>;
+export declare function parseRegistry(value: unknown): LookoutRegistry;

package/dist/src/registry.js

@@ -0,0 +1,141 @@
+import { readFile } from "node:fs/promises";
+import path from "node:path";
+export class RegistryValidationError extends Error {
+    issues;
+    constructor(issues) {
+        super(`Invalid Lookout registry:\n${issues.map((issue) => `- ${issue}`).join("\n")}`);
+        this.name = "RegistryValidationError";
+        this.issues = issues;
+    }
+}
+export class LookoutRegistry {
+    version = 1;
+    sources;
+    #byId;
+    constructor(sources) {
+        this.sources = sources;
+        this.#byId = new Map(sources.map((source) => [source.id, source]));
+    }
+    list() {
+        return this.sources;
+    }
+    get(id) {
+        return this.#byId.get(id);
+    }
+}
+export async function loadRegistry(registryPath = path.join(process.cwd(), "lookout.sources.json")) {
+    const raw = await readFile(registryPath, "utf8");
+    let parsed;
+    try {
+        parsed = JSON.parse(raw);
+    }
+    catch (error) {
+        const message = error instanceof Error ? error.message : String(error);
+        throw new RegistryValidationError([`document is not valid JSON: ${message}`]);
+    }
+    return parseRegistry(parsed);
+}
+export function parseRegistry(value) {
+    const issues = [];
+    if (!isRecord(value)) {
+        throw new RegistryValidationError(["document must be an object"]);
+    }
+    if (value.version !== 1)
+        issues.push("version must be exactly 1");
+    if (!Array.isArray(value.sources)) {
+        issues.push("sources must be an array");
+        throw new RegistryValidationError(issues);
+    }
+    const ids = new Map();
+    const sources = [];
+    value.sources.forEach((candidate, index) => {
+        const context = `sources[${index}]`;
+        if (!isRecord(candidate)) {
+            issues.push(`${context} must be an object`);
+            return;
+        }
+        const id = candidate.id;
+        const label = typeof id === "string" && id.trim() ? `${context} (id "${id}")` : context;
+        if (typeof id !== "string" || id.trim() === "") {
+            issues.push(`${context}.id must be a non-empty string`);
+        }
+        else {
+            const first = ids.get(id);
+            if (first !== undefined)
+                issues.push(`${label}.id duplicates sources[${first}].id`);
+            else
+                ids.set(id, index);
+        }
+        if (candidate.kind !== "web-page" && candidate.kind !== "api-record") {
+            issues.push(`${label}.kind must be "web-page" or "api-record"`);
+        }
+        if (!isHttpUrl(candidate.url)) {
+            issues.push(`${label}.url must be an absolute HTTP(S) URL`);
+        }
+        if (typeof candidate.cadenceHint !== "string" || candidate.cadenceHint.trim() === "") {
+            issues.push(`${label}.cadenceHint must be a non-empty string`);
+        }
+        if (candidate.renderPolicy !== "never" &&
+            candidate.renderPolicy !== "on-shell-warning" &&
+            candidate.renderPolicy !== "always") {
+            issues.push(`${label}.renderPolicy must be "never", "on-shell-warning", or "always"`);
+        }
+        validateTargetSchema(candidate.targetSchema, label, issues);
+        if (typeof id === "string" && id.trim() !== "" &&
+            (candidate.kind === "web-page" || candidate.kind === "api-record") &&
+            isHttpUrl(candidate.url) &&
+            typeof candidate.cadenceHint === "string" && candidate.cadenceHint.trim() !== "" &&
+            (candidate.renderPolicy === "never" || candidate.renderPolicy === "on-shell-warning" || candidate.renderPolicy === "always") &&
+            Array.isArray(candidate.targetSchema)) {
+            sources.push(candidate);
+        }
+    });
+    if (issues.length > 0)
+        throw new RegistryValidationError(issues);
+    return new LookoutRegistry(sources);
+}
+function validateTargetSchema(value, context, issues) {
+    if (!Array.isArray(value)) {
+        issues.push(`${context}.targetSchema must be an array`);
+        return;
+    }
+    const allowedTypes = new Set(["string", "number", "boolean", "date", "enum", "array", "object"]);
+    value.forEach((field, index) => {
+        const label = `${context}.targetSchema[${index}]`;
+        if (!isRecord(field)) {
+            issues.push(`${label} must be an object`);
+            return;
+        }
+        if (typeof field.path !== "string" || field.path.trim() === "")
+            issues.push(`${label}.path must be a non-empty string`);
+        if (typeof field.type !== "string" || !allowedTypes.has(field.type))
+            issues.push(`${label}.type is invalid`);
+        if (field.enumValues !== undefined && (!Array.isArray(field.enumValues) || field.enumValues.some((item) => typeof item !== "string"))) {
+            issues.push(`${label}.enumValues must be an array of strings`);
+        }
+        if (field.type === "enum" && (!Array.isArray(field.enumValues) || field.enumValues.length === 0)) {
+            issues.push(`${label}.enumValues must be non-empty for enum fields`);
+        }
+        if (field.description !== undefined && typeof field.description !== "string")
+            issues.push(`${label}.description must be a string`);
+        if (field.required !== undefined && typeof field.required !== "boolean")
+            issues.push(`${label}.required must be a boolean`);
+        if (field.inferenceType !== undefined && field.inferenceType !== "explicit" && field.inferenceType !== "inferred") {
+            issues.push(`${label}.inferenceType must be "explicit" or "inferred"`);
+        }
+    });
+}
+function isRecord(value) {
+    return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+function isHttpUrl(value) {
+    if (typeof value !== "string")
+        return false;
+    try {
+        const url = new URL(value);
+        return (url.protocol === "http:" || url.protocol === "https:") && Boolean(url.hostname);
+    }
+    catch {
+        return false;
+    }
+}

package/dist/src/snapshot-store.d.ts

@@ -0,0 +1,2 @@
+import type { SnapshotStore } from "@kontourai/traverse/fetch";
+export declare function createLookoutSnapshotStore(root?: string): SnapshotStore;

package/dist/src/snapshot-store.js

@@ -0,0 +1,5 @@
+import path from "node:path";
+import { createFilesystemSnapshotStore } from "@kontourai/traverse/fetch";
+export function createLookoutSnapshotStore(root = path.join(process.cwd(), ".kontourai", "lookout", "snapshots")) {
+    return createFilesystemSnapshotStore({ root });
+}

package/dist/src/structural-diff.d.ts

@@ -0,0 +1,24 @@
+import { type DiffResult, type IdentityResult } from "./canonical-value.js";
+export interface StructuralFacts<T> {
+    readonly valueChanged: boolean;
+    readonly provenanceChanged: boolean;
+}
+export type StructuralComparison<T> = DiffResult<StructuralFacts<T>>;
+export interface StructuralComparisonOptions<T> {
+    readonly value?: (item: T) => unknown;
+    readonly provenance?: (item: T) => unknown;
+}
+export declare function compareStructural<T>(prior: T, current: T, options?: StructuralComparisonOptions<T>): StructuralComparison<T>;
+export interface KeyedMultisetFacts<T, K extends string> {
+    readonly unchanged: boolean;
+    readonly retained: readonly {
+        readonly prior: T;
+        readonly current: T;
+    }[];
+    readonly additions: readonly T[];
+    readonly removals: readonly T[];
+}
+export interface KeyedMultisetOptions<T, K extends string> {
+    readonly identity: (item: T) => K | IdentityResult<K>;
+}
+export declare function diffKeyedMultiset<T, K extends string>(prior: readonly T[], current: readonly T[], options: KeyedMultisetOptions<T, K>): DiffResult<KeyedMultisetFacts<T, K>>;

package/dist/src/structural-diff.js

@@ -0,0 +1,94 @@
+import { canonicalValueKey, } from "./canonical-value.js";
+function callbackError(label, cause) {
+    return { kind: "callback-threw", message: `${label} callback threw`, cause };
+}
+function projected(label, projector, item) {
+    try {
+        return { ok: true, value: projector(item) };
+    }
+    catch (cause) {
+        return { ok: false, error: callbackError(label, cause) };
+    }
+}
+function equalCanonical(prior, current) {
+    const priorKey = canonicalValueKey(prior);
+    if (!priorKey.ok)
+        return priorKey;
+    const currentKey = canonicalValueKey(current);
+    if (!currentKey.ok)
+        return currentKey;
+    return { ok: true, value: priorKey.key === currentKey.key };
+}
+export function compareStructural(prior, current, options = {}) {
+    const valueProjector = options.value ?? ((item) => item);
+    const priorValue = projected("value", valueProjector, prior);
+    if (!priorValue.ok)
+        return priorValue;
+    const currentValue = projected("value", valueProjector, current);
+    if (!currentValue.ok)
+        return currentValue;
+    const valueEqual = equalCanonical(priorValue.value, currentValue.value);
+    if (!valueEqual.ok)
+        return valueEqual;
+    let provenanceChanged = false;
+    if (options.provenance) {
+        const priorProvenance = projected("provenance", options.provenance, prior);
+        if (!priorProvenance.ok)
+            return priorProvenance;
+        const currentProvenance = projected("provenance", options.provenance, current);
+        if (!currentProvenance.ok)
+            return currentProvenance;
+        const provenanceEqual = equalCanonical(priorProvenance.value, currentProvenance.value);
+        if (!provenanceEqual.ok)
+            return provenanceEqual;
+        provenanceChanged = !provenanceEqual.value;
+    }
+    return { ok: true, value: { valueChanged: !valueEqual.value, provenanceChanged } };
+}
+function identityOf(callback, item) {
+    try {
+        const result = callback(item);
+        if (typeof result === "string")
+            return { ok: true, value: result };
+        return result.ok ? { ok: true, value: result.key } : result;
+    }
+    catch (cause) {
+        return { ok: false, error: callbackError("identity", cause) };
+    }
+}
+export function diffKeyedMultiset(prior, current, options) {
+    const currentKeys = [];
+    for (const item of current) {
+        const identity = identityOf(options.identity, item);
+        if (!identity.ok)
+            return identity;
+        currentKeys.push(identity.value);
+    }
+    const queues = new Map();
+    for (let index = 0; index < currentKeys.length; index += 1) {
+        const key = currentKeys[index];
+        const queue = queues.get(key) ?? [];
+        queue.push(index);
+        queues.set(key, queue);
+    }
+    const retained = [];
+    const removals = [];
+    const matchedCurrent = new Set();
+    for (const item of prior) {
+        const identity = identityOf(options.identity, item);
+        if (!identity.ok)
+            return identity;
+        const currentIndex = queues.get(identity.value)?.shift();
+        if (currentIndex === undefined)
+            removals.push(item);
+        else {
+            matchedCurrent.add(currentIndex);
+            retained.push({ prior: item, current: current[currentIndex] });
+        }
+    }
+    const additions = current.filter((_item, index) => !matchedCurrent.has(index));
+    return {
+        ok: true,
+        value: { unchanged: additions.length === 0 && removals.length === 0, retained, additions, removals },
+    };
+}

package/dist/src/survey-emission.d.ts

@@ -0,0 +1,68 @@
+import { type SurveyInput } from "@kontourai/survey";
+import type { LookoutSource } from "./registry.js";
+import { type ProposalDiffEvent, type ProposalSetDiff, type ProposalSetDiffInput, type ProposalSetFacts, type ProposalSetObservation } from "./proposal-diff.js";
+import type { ObservationCheckAnchor, ObservationStore, StoredProposalObservationV1 } from "./observation-store.js";
+export interface BaselineEstablishedFact {
+    readonly kind: "baseline-established";
+    readonly sourceId: string;
+    readonly snapshotRef: string;
+    readonly observedAt: string;
+    readonly origin: LookoutSource["kind"];
+    readonly resolution: "observation";
+    readonly proposalCount: number;
+}
+export type EmissionFact = BaselineEstablishedFact | {
+    readonly kind: "proposal-set-facts";
+    readonly priorSnapshotRef: string;
+    readonly currentSnapshotRef: string;
+    readonly origin: LookoutSource["kind"];
+    readonly resolution: "observation";
+    readonly value: ProposalSetFacts;
+};
+export interface EmissionSuccess {
+    readonly sourceId: string;
+    readonly events: readonly ProposalDiffEvent[];
+    readonly facts: readonly EmissionFact[];
+    readonly surveyInput: SurveyInput | null;
+    readonly committedObservation: StoredProposalObservationV1;
+    readonly warnings: readonly string[];
+}
+export type EmissionErrorKind = "invalid-input" | "prior-state-error" | "diff-error" | "authoring-error" | "persistence-error";
+export interface EmissionError {
+    readonly kind: EmissionErrorKind;
+    readonly message: string;
+    readonly cause?: unknown;
+}
+export type EmissionResult = {
+    readonly ok: true;
+    readonly value: EmissionSuccess;
+} | {
+    readonly ok: false;
+    readonly error: EmissionError;
+};
+export interface EmitSurveyInput<E> {
+    readonly source: LookoutSource;
+    readonly current: ProposalSetObservation;
+    readonly check: ObservationCheckAnchor;
+    readonly callbacks: Omit<ProposalSetDiffInput<E>, "prior" | "current">;
+}
+export interface SurveyEmitter<E> {
+    emit(input: EmitSurveyInput<E>): Promise<EmissionResult>;
+}
+export interface CreateSurveyEmitterOptions<E> {
+    readonly store: ObservationStore;
+    readonly now?: () => string;
+    readonly diff?: (input: ProposalSetDiffInput<E>) => {
+        readonly ok: true;
+        readonly value: ProposalSetDiff;
+    } | {
+        readonly ok: false;
+        readonly error: {
+            readonly message: string;
+        };
+    };
+    /** Test seam. Production callers must leave this as observation. */
+    readonly resolution?: "observation";
+    readonly transformSurveyInput?: (input: SurveyInput) => SurveyInput;
+}
+export declare function createSurveyEmitter<E>(options: CreateSurveyEmitterOptions<E>): SurveyEmitter<E>;