@wirecat/cli-core 0.558.0 → 0.560.0

package/dist/domain/records.js

@@ -0,0 +1,234 @@
+import { createHash } from "node:crypto";
+import { holdsCredential, secretNamed } from "../runs/secrets.js";
+export const RECORDS_SCHEMA_VERSION = 1;
+export const RECORD_OPERATIONS = ["upsert", "delete"];
+/** Types with a table of their own; anything else a source sends is `custom:<name>` and stays a record only. */
+export const RECORD_ENTITY_TYPES = ["invoice", "payment", "person", "organization", "project"];
+const WORD = "^[a-z0-9][a-z0-9_-]{0,62}$";
+const TYPE = "^(invoice|payment|person|organization|project|custom:[a-z0-9][a-z0-9_-]{0,62})$";
+const MAX_ID = 512;
+const MAX_FIELDS_BYTES = 65_536;
+const MAX_RELATIONS = 100;
+/** The v1 envelope as JSON Schema (Draft 2020-12); `schemas/records-envelope.v1.schema.json` is this, written out. */
+export const RECORDS_ENVELOPE_SCHEMA = {
+    $schema: "https://json-schema.org/draft/2020-12/schema",
+    title: "WireCat records envelope, version 1",
+    description: "Records one source sends to the store. JSONL is the same model: the first line holds schema_version and " +
+        "source, each further line one record.",
+    type: "object",
+    required: ["schema_version", "source", "records"],
+    additionalProperties: false,
+    properties: {
+        schema_version: { const: RECORDS_SCHEMA_VERSION },
+        source: { $ref: "#/$defs/source" },
+        records: { type: "array", items: { $ref: "#/$defs/record" } },
+    },
+    $defs: {
+        source: {
+            type: "object",
+            required: ["provider", "account", "batch_id"],
+            additionalProperties: false,
+            properties: {
+                provider: { type: "string", pattern: WORD, description: "The system the records come from" },
+                account: { type: "string", minLength: 1, maxLength: MAX_ID, description: "The account in that system" },
+                batch_id: { type: "string", minLength: 1, maxLength: MAX_ID, description: "This import, for the receipt" },
+            },
+        },
+        record: {
+            type: "object",
+            required: ["key", "type", "external_id", "operation"],
+            additionalProperties: false,
+            properties: {
+                key: { type: "string", minLength: 1, maxLength: MAX_ID },
+                type: { type: "string", pattern: TYPE },
+                external_id: { type: "string", minLength: 1, maxLength: MAX_ID },
+                operation: { enum: [...RECORD_OPERATIONS] },
+                fields: {
+                    type: "object",
+                    description: "What the source says, as data: never secrets, never instructions",
+                },
+                revision: { type: "integer", minimum: 0, maximum: Number.MAX_SAFE_INTEGER },
+                relations: { type: "array", maxItems: MAX_RELATIONS, items: { $ref: "#/$defs/relation" } },
+            },
+        },
+        relation: {
+            type: "object",
+            required: ["role", "type", "external_id"],
+            additionalProperties: false,
+            properties: {
+                role: { type: "string", pattern: WORD },
+                type: { type: "string", pattern: TYPE },
+                external_id: { type: "string", minLength: 1, maxLength: MAX_ID },
+            },
+        },
+    },
+};
+const wordPattern = new RegExp(WORD);
+const typePattern = new RegExp(TYPE);
+const isObject = (value) => typeof value === "object" && value !== null && !Array.isArray(value);
+const escapePointer = (segment) => String(segment).replaceAll("~", "~0").replaceAll("/", "~1");
+export const pointerOf = (...segments) => segments.map((s) => `/${escapePointer(s)}`).join("");
+const idText = (value) => typeof value === "string" && value.trim() !== "" && value.length <= MAX_ID && !value.includes("\0");
+const extraKeys = (value, allowed) => Object.keys(value).filter((key) => !allowed.includes(key));
+/** The first secret in `fields`, as a pointer below it: a secret-named key, or a value in a known credential shape. */
+const secretIn = (value, at) => {
+    if (typeof value === "string")
+        return holdsCredential(value) ? at : undefined;
+    if (Array.isArray(value)) {
+        for (const [index, item] of value.entries()) {
+            const found = secretIn(item, [...at, index]);
+            if (found)
+                return found;
+        }
+        return undefined;
+    }
+    if (!isObject(value))
+        return undefined;
+    for (const [key, item] of Object.entries(value)) {
+        if (secretNamed(key))
+            return [...at, key];
+        const found = secretIn(item, [...at, key]);
+        if (found)
+            return found;
+    }
+    return undefined;
+};
+const relationErrors = (relations, at) => {
+    if (relations === undefined)
+        return [];
+    if (!Array.isArray(relations) || relations.length > MAX_RELATIONS)
+        return [invalid(at, `relations is a list of at most ${MAX_RELATIONS}`)];
+    return relations.flatMap((relation, index) => {
+        const here = [...at, index];
+        if (!isObject(relation))
+            return [invalid(here, "a relation is an object")];
+        const extra = extraKeys(relation, ["role", "type", "external_id"]);
+        if (extra.length > 0)
+            return [invalid([...here, extra[0]], "a relation holds role, type and external_id")];
+        if (typeof relation.role !== "string" || !wordPattern.test(relation.role))
+            return [invalid([...here, "role"], "a role is lowercase letters, digits, - and _")];
+        if (typeof relation.type !== "string" || !typePattern.test(relation.type))
+            return [invalid([...here, "type"], `a type is one of ${RECORD_ENTITY_TYPES.join(", ")} or custom:<name>`)];
+        if (!idText(relation.external_id))
+            return [invalid([...here, "external_id"], `an external id takes 1–${MAX_ID} characters`)];
+        return [];
+    });
+};
+const invalid = (at, message, code = "record.invalid") => ({
+    pointer: pointerOf(...at),
+    code,
+    message,
+});
+/** Why one record cannot be taken, or nothing; `index` is its place in `records`. */
+export const recordErrors = (record, index) => {
+    const at = ["records", index];
+    if (!isObject(record))
+        return [invalid(at, "a record is an object")];
+    const key = typeof record.key === "string" ? record.key : undefined;
+    const withKey = (errors) => errors.map((error) => (key === undefined ? error : { ...error, key }));
+    const extra = extraKeys(record, ["key", "type", "external_id", "operation", "fields", "revision", "relations"]);
+    if (extra.length > 0)
+        return withKey([invalid([...at, extra[0]], `a record has no ${extra[0]}`)]);
+    if (!idText(record.key))
+        return [invalid([...at, "key"], `a key takes 1–${MAX_ID} characters`)];
+    if (typeof record.type !== "string" || !typePattern.test(record.type))
+        return withKey([
+            invalid([...at, "type"], `a type is one of ${RECORD_ENTITY_TYPES.join(", ")} or custom:<name>`, "record.type"),
+        ]);
+    if (!idText(record.external_id))
+        return withKey([invalid([...at, "external_id"], `an external id takes 1–${MAX_ID} characters`)]);
+    if (!RECORD_OPERATIONS.includes(record.operation))
+        return withKey([invalid([...at, "operation"], "an operation is upsert or delete")]);
+    if (record.revision !== undefined &&
+        (typeof record.revision !== "number" || !Number.isSafeInteger(record.revision) || record.revision < 0))
+        return withKey([invalid([...at, "revision"], "a revision is a whole number, 0 or more")]);
+    if (record.operation === "upsert" && !isObject(record.fields))
+        return withKey([invalid([...at, "fields"], "an upsert carries its fields as an object")]);
+    if (record.fields !== undefined && !isObject(record.fields))
+        return withKey([invalid([...at, "fields"], "fields is an object")]);
+    if (record.fields !== undefined && Buffer.byteLength(JSON.stringify(record.fields)) > MAX_FIELDS_BYTES)
+        return withKey([invalid([...at, "fields"], `fields take at most ${MAX_FIELDS_BYTES} bytes`, "record.too-large")]);
+    const secret = record.fields === undefined ? undefined : secretIn(record.fields, [...at, "fields"]);
+    if (secret)
+        return withKey([
+            invalid(secret, "a secret never goes into the store: drop this field at the source", "record.secret-field"),
+        ]);
+    return withKey(relationErrors(record.relations, [...at, "relations"]));
+};
+/** The envelope's own errors (not its records'): when there is one, no record is taken. */
+export const envelopeErrors = (envelope, account) => {
+    if (!isObject(envelope))
+        return [invalid([], "an envelope is an object", "envelope.invalid")];
+    const extra = extraKeys(envelope, ["schema_version", "source", "records"]);
+    if (extra.length > 0)
+        return [invalid([extra[0]], `an envelope has no ${extra[0]}`, "envelope.invalid")];
+    if (envelope.schema_version !== RECORDS_SCHEMA_VERSION)
+        return [
+            invalid(["schema_version"], `this build reads schema_version ${RECORDS_SCHEMA_VERSION}, not ${JSON.stringify(envelope.schema_version)}`, "envelope.schema-version"),
+        ];
+    const { source } = envelope;
+    if (!isObject(source))
+        return [invalid(["source"], "source is an object", "envelope.invalid")];
+    const sourceExtra = extraKeys(source, ["provider", "account", "batch_id"]);
+    if (sourceExtra.length > 0)
+        return [
+            invalid(["source", sourceExtra[0]], "source holds provider, account and batch_id", "envelope.invalid"),
+        ];
+    if (typeof source.provider !== "string" || !wordPattern.test(source.provider))
+        return [invalid(["source", "provider"], "a provider is lowercase letters, digits, - and _", "envelope.invalid")];
+    if (!idText(source.account))
+        return [invalid(["source", "account"], `an account takes 1–${MAX_ID} characters`, "envelope.invalid")];
+    if (!idText(source.batch_id))
+        return [invalid(["source", "batch_id"], `a batch id takes 1–${MAX_ID} characters`, "envelope.invalid")];
+    if (source.account !== account)
+        return [
+            invalid(["source", "account"], `the envelope is for account ${JSON.stringify(source.account)}, the import for ${JSON.stringify(account)}`, "envelope.account-mismatch"),
+        ];
+    if (!Array.isArray(envelope.records))
+        return [invalid(["records"], "records is a list", "envelope.invalid")];
+    return [];
+};
+const canonical = (value) => {
+    if (Array.isArray(value))
+        return value.map(canonical);
+    if (!isObject(value))
+        return value;
+    return Object.fromEntries(Object.keys(value)
+        .sort()
+        .map((key) => [key, canonical(value[key])]));
+};
+/** The same fields and relations in any key or relation order give the same fingerprint. */
+export const recordFingerprint = (fields, relations) => createHash("sha256")
+    .update(JSON.stringify(canonical({
+    fields,
+    relations: relations
+        .map((relation) => [relation.role, relation.type, relation.external_id])
+        .sort((a, b) => a.join("\0").localeCompare(b.join("\0"))),
+})))
+    .digest("hex");
+/**
+ * A JSONL import as the envelope it stands for: the first line holds `schema_version` and `source`, each further
+ * non-empty line one record. A line that is not JSON comes back as a record the envelope check refuses, at its index.
+ */
+export const envelopeOfJsonl = (text) => {
+    const lines = text.split(/\r?\n/);
+    const first = lines.findIndex((line) => line.trim() !== "");
+    if (first < 0)
+        return {};
+    const parse = (line) => {
+        try {
+            return JSON.parse(line);
+        }
+        catch {
+            return line;
+        }
+    };
+    const header = parse(lines[first]);
+    const records = [];
+    for (let at = first + 1; at < lines.length; at++) {
+        const line = lines[at];
+        if (line.trim() !== "")
+            records.push(parse(line));
+    }
+    return isObject(header) ? { ...header, records } : header;
+};

package/dist/entities/contact-context.d.ts

@@ -349,6 +349,9 @@ export declare const contactMessages: v.StrictObjectSchema<{
                 readonly bulk: v.OptionalSchema<v.BooleanSchema<undefined>, undefined>;
                 readonly list_id: v.OptionalSchema<v.NullableSchema<v.StringSchema<undefined>, undefined>, undefined>;
                 readonly rule_id: v.OptionalSchema<v.NullableSchema<v.NumberSchema<undefined>, undefined>, undefined>;
+                readonly fingerprint: v.OptionalSchema<v.SchemaWithPipe<readonly [v.StringSchema<undefined>, v.RegexAction<string, undefined>]>, undefined>;
+                readonly new_sender: v.OptionalSchema<v.BooleanSchema<undefined>, undefined>;
+                readonly sender_identities: v.OptionalSchema<v.NumberSchema<undefined>, undefined>;
                 readonly reply_to: v.ArraySchema<v.StrictObjectSchema<{
                     readonly name: v.NullableSchema<v.StringSchema<undefined>, undefined>;
                     readonly address: v.StringSchema<undefined>;
@@ -1124,6 +1127,9 @@ export declare const contactContext: v.UnionSchema<[v.StrictObjectSchema<{
                 readonly bulk: v.OptionalSchema<v.BooleanSchema<undefined>, undefined>;
                 readonly list_id: v.OptionalSchema<v.NullableSchema<v.StringSchema<undefined>, undefined>, undefined>;
                 readonly rule_id: v.OptionalSchema<v.NullableSchema<v.NumberSchema<undefined>, undefined>, undefined>;
+                readonly fingerprint: v.OptionalSchema<v.SchemaWithPipe<readonly [v.StringSchema<undefined>, v.RegexAction<string, undefined>]>, undefined>;
+                readonly new_sender: v.OptionalSchema<v.BooleanSchema<undefined>, undefined>;
+                readonly sender_identities: v.OptionalSchema<v.NumberSchema<undefined>, undefined>;
                 readonly reply_to: v.ArraySchema<v.StrictObjectSchema<{
                     readonly name: v.NullableSchema<v.StringSchema<undefined>, undefined>;
                     readonly address: v.StringSchema<undefined>;

package/dist/entities/conversation-results.d.ts

@@ -227,6 +227,9 @@ export declare const conversationDetails: v.UnionSchema<[v.StrictObjectSchema<{
             readonly bulk: v.OptionalSchema<v.BooleanSchema<undefined>, undefined>;
             readonly list_id: v.OptionalSchema<v.NullableSchema<v.StringSchema<undefined>, undefined>, undefined>;
             readonly rule_id: v.OptionalSchema<v.NullableSchema<v.NumberSchema<undefined>, undefined>, undefined>;
+            readonly fingerprint: v.OptionalSchema<v.SchemaWithPipe<readonly [v.StringSchema<undefined>, v.RegexAction<string, undefined>]>, undefined>;
+            readonly new_sender: v.OptionalSchema<v.BooleanSchema<undefined>, undefined>;
+            readonly sender_identities: v.OptionalSchema<v.NumberSchema<undefined>, undefined>;
         }, undefined>, undefined>;
         readonly provider_metadata: v.OptionalSchema<v.RecordSchema<v.StringSchema<undefined>, v.UnknownSchema, undefined>, undefined>;
     }, undefined>, undefined>;
@@ -447,6 +450,9 @@ export declare const conversationDetails: v.UnionSchema<[v.StrictObjectSchema<{
         readonly bulk: v.OptionalSchema<v.BooleanSchema<undefined>, undefined>;
         readonly list_id: v.OptionalSchema<v.NullableSchema<v.StringSchema<undefined>, undefined>, undefined>;
         readonly rule_id: v.OptionalSchema<v.NullableSchema<v.NumberSchema<undefined>, undefined>, undefined>;
+        readonly fingerprint: v.OptionalSchema<v.SchemaWithPipe<readonly [v.StringSchema<undefined>, v.RegexAction<string, undefined>]>, undefined>;
+        readonly new_sender: v.OptionalSchema<v.BooleanSchema<undefined>, undefined>;
+        readonly sender_identities: v.OptionalSchema<v.NumberSchema<undefined>, undefined>;
     }, undefined>, undefined>;
     readonly provider_metadata: v.OptionalSchema<v.RecordSchema<v.StringSchema<undefined>, v.UnknownSchema, undefined>, undefined>;
 }, undefined>], undefined>;

package/dist/entities/evidence.d.ts

@@ -218,6 +218,9 @@ export declare const contribution: v.StrictObjectSchema<{
             readonly bulk: v.OptionalSchema<v.BooleanSchema<undefined>, undefined>;
             readonly list_id: v.OptionalSchema<v.NullableSchema<v.StringSchema<undefined>, undefined>, undefined>;
             readonly rule_id: v.OptionalSchema<v.NullableSchema<v.NumberSchema<undefined>, undefined>, undefined>;
+            readonly fingerprint: v.OptionalSchema<v.SchemaWithPipe<readonly [v.StringSchema<undefined>, v.RegexAction<string, undefined>]>, undefined>;
+            readonly new_sender: v.OptionalSchema<v.BooleanSchema<undefined>, undefined>;
+            readonly sender_identities: v.OptionalSchema<v.NumberSchema<undefined>, undefined>;
         }, undefined>, undefined>;
         readonly provider_metadata: v.OptionalSchema<v.RecordSchema<v.StringSchema<undefined>, v.UnknownSchema, undefined>, undefined>;
     }, undefined>, "entries" | "~run" | "~standard" | "~types"> & {
@@ -438,6 +441,9 @@ export declare const contribution: v.StrictObjectSchema<{
                 readonly bulk: v.OptionalSchema<v.BooleanSchema<undefined>, undefined>;
                 readonly list_id: v.OptionalSchema<v.NullableSchema<v.StringSchema<undefined>, undefined>, undefined>;
                 readonly rule_id: v.OptionalSchema<v.NullableSchema<v.NumberSchema<undefined>, undefined>, undefined>;
+                readonly fingerprint: v.OptionalSchema<v.SchemaWithPipe<readonly [v.StringSchema<undefined>, v.RegexAction<string, undefined>]>, undefined>;
+                readonly new_sender: v.OptionalSchema<v.BooleanSchema<undefined>, undefined>;
+                readonly sender_identities: v.OptionalSchema<v.NumberSchema<undefined>, undefined>;
             }, undefined>, undefined>;
             readonly provider_metadata: v.OptionalSchema<v.RecordSchema<v.StringSchema<undefined>, v.UnknownSchema, undefined>, undefined>;
         }, "chat_title" | "ref" | "sender_name" | "text" | "timestamp">;
@@ -696,6 +702,9 @@ export declare const contribution: v.StrictObjectSchema<{
             readonly bulk: v.OptionalSchema<v.BooleanSchema<undefined>, undefined>;
             readonly list_id: v.OptionalSchema<v.NullableSchema<v.StringSchema<undefined>, undefined>, undefined>;
             readonly rule_id: v.OptionalSchema<v.NullableSchema<v.NumberSchema<undefined>, undefined>, undefined>;
+            readonly fingerprint: v.OptionalSchema<v.SchemaWithPipe<readonly [v.StringSchema<undefined>, v.RegexAction<string, undefined>]>, undefined>;
+            readonly new_sender: v.OptionalSchema<v.BooleanSchema<undefined>, undefined>;
+            readonly sender_identities: v.OptionalSchema<v.NumberSchema<undefined>, undefined>;
         }, undefined>, undefined>;
         readonly provider_metadata: v.OptionalSchema<v.RecordSchema<v.StringSchema<undefined>, v.UnknownSchema, undefined>, undefined>;
     }, undefined>, "entries" | "~run" | "~standard" | "~types"> & {
@@ -916,6 +925,9 @@ export declare const contribution: v.StrictObjectSchema<{
                 readonly bulk: v.OptionalSchema<v.BooleanSchema<undefined>, undefined>;
                 readonly list_id: v.OptionalSchema<v.NullableSchema<v.StringSchema<undefined>, undefined>, undefined>;
                 readonly rule_id: v.OptionalSchema<v.NullableSchema<v.NumberSchema<undefined>, undefined>, undefined>;
+                readonly fingerprint: v.OptionalSchema<v.SchemaWithPipe<readonly [v.StringSchema<undefined>, v.RegexAction<string, undefined>]>, undefined>;
+                readonly new_sender: v.OptionalSchema<v.BooleanSchema<undefined>, undefined>;
+                readonly sender_identities: v.OptionalSchema<v.NumberSchema<undefined>, undefined>;
             }, undefined>, undefined>;
             readonly provider_metadata: v.OptionalSchema<v.RecordSchema<v.StringSchema<undefined>, v.UnknownSchema, undefined>, undefined>;
         }, "chat_title" | "ref" | "sender_name" | "text" | "timestamp">;
@@ -1210,6 +1222,9 @@ export declare const evidence: v.UnionSchema<[v.StrictObjectSchema<{
             readonly bulk: v.OptionalSchema<v.BooleanSchema<undefined>, undefined>;
             readonly list_id: v.OptionalSchema<v.NullableSchema<v.StringSchema<undefined>, undefined>, undefined>;
             readonly rule_id: v.OptionalSchema<v.NullableSchema<v.NumberSchema<undefined>, undefined>, undefined>;
+            readonly fingerprint: v.OptionalSchema<v.SchemaWithPipe<readonly [v.StringSchema<undefined>, v.RegexAction<string, undefined>]>, undefined>;
+            readonly new_sender: v.OptionalSchema<v.BooleanSchema<undefined>, undefined>;
+            readonly sender_identities: v.OptionalSchema<v.NumberSchema<undefined>, undefined>;
         }, undefined>, undefined>;
         readonly provider_metadata: v.OptionalSchema<v.RecordSchema<v.StringSchema<undefined>, v.UnknownSchema, undefined>, undefined>;
     }, undefined>, "entries" | "~run" | "~standard" | "~types"> & {
@@ -1430,6 +1445,9 @@ export declare const evidence: v.UnionSchema<[v.StrictObjectSchema<{
                 readonly bulk: v.OptionalSchema<v.BooleanSchema<undefined>, undefined>;
                 readonly list_id: v.OptionalSchema<v.NullableSchema<v.StringSchema<undefined>, undefined>, undefined>;
                 readonly rule_id: v.OptionalSchema<v.NullableSchema<v.NumberSchema<undefined>, undefined>, undefined>;
+                readonly fingerprint: v.OptionalSchema<v.SchemaWithPipe<readonly [v.StringSchema<undefined>, v.RegexAction<string, undefined>]>, undefined>;
+                readonly new_sender: v.OptionalSchema<v.BooleanSchema<undefined>, undefined>;
+                readonly sender_identities: v.OptionalSchema<v.NumberSchema<undefined>, undefined>;
             }, undefined>, undefined>;
             readonly provider_metadata: v.OptionalSchema<v.RecordSchema<v.StringSchema<undefined>, v.UnknownSchema, undefined>, undefined>;
         }, "chat_title" | "ref" | "sender_name" | "text" | "timestamp">;
@@ -1688,6 +1706,9 @@ export declare const evidence: v.UnionSchema<[v.StrictObjectSchema<{
             readonly bulk: v.OptionalSchema<v.BooleanSchema<undefined>, undefined>;
             readonly list_id: v.OptionalSchema<v.NullableSchema<v.StringSchema<undefined>, undefined>, undefined>;
             readonly rule_id: v.OptionalSchema<v.NullableSchema<v.NumberSchema<undefined>, undefined>, undefined>;
+            readonly fingerprint: v.OptionalSchema<v.SchemaWithPipe<readonly [v.StringSchema<undefined>, v.RegexAction<string, undefined>]>, undefined>;
+            readonly new_sender: v.OptionalSchema<v.BooleanSchema<undefined>, undefined>;
+            readonly sender_identities: v.OptionalSchema<v.NumberSchema<undefined>, undefined>;
         }, undefined>, undefined>;
         readonly provider_metadata: v.OptionalSchema<v.RecordSchema<v.StringSchema<undefined>, v.UnknownSchema, undefined>, undefined>;
     }, undefined>, "entries" | "~run" | "~standard" | "~types"> & {
@@ -1908,6 +1929,9 @@ export declare const evidence: v.UnionSchema<[v.StrictObjectSchema<{
                 readonly bulk: v.OptionalSchema<v.BooleanSchema<undefined>, undefined>;
                 readonly list_id: v.OptionalSchema<v.NullableSchema<v.StringSchema<undefined>, undefined>, undefined>;
                 readonly rule_id: v.OptionalSchema<v.NullableSchema<v.NumberSchema<undefined>, undefined>, undefined>;
+                readonly fingerprint: v.OptionalSchema<v.SchemaWithPipe<readonly [v.StringSchema<undefined>, v.RegexAction<string, undefined>]>, undefined>;
+                readonly new_sender: v.OptionalSchema<v.BooleanSchema<undefined>, undefined>;
+                readonly sender_identities: v.OptionalSchema<v.NumberSchema<undefined>, undefined>;
             }, undefined>, undefined>;
             readonly provider_metadata: v.OptionalSchema<v.RecordSchema<v.StringSchema<undefined>, v.UnknownSchema, undefined>, undefined>;
         }, "chat_title" | "ref" | "sender_name" | "text" | "timestamp">;