@airfone/connector 0.0.0-stage → 0.1.0

package/dist/host.js

@@ -0,0 +1,219 @@
+// Host calls for local runs. LocalHost is `dev`: real network (declared hosts
+// only), a store kept in .airfone/store.json, records printed. ReviewHost is
+// `test` and matches the server's review sandbox: no network, an in-memory
+// store, records accepted and dropped.
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
+import { dirname } from 'node:path';
+import { applyAuth } from './http.js';
+import { createHash, createHmac } from 'node:crypto';
+import { allowedHosts, secretNames } from './manifest.js';
+import { fetchSync } from './runner.js';
+const MAX_KV_VALUE = 64 * 1024;
+function pairs(v) {
+    if (!v || typeof v !== 'object')
+        return [];
+    return Object.entries(v).map(([k, x]) => [k, typeof x === 'string' ? x : JSON.stringify(x)]);
+}
+class BaseHost {
+    m;
+    config;
+    granted;
+    events = [];
+    constructor(m, config, granted) {
+        this.m = m;
+        this.config = config;
+        this.granted = granted;
+    }
+    call(op, args, deadline) {
+        const str = (k) => { const v = args?.[k]; if (typeof v !== 'string')
+            throw `${op}: ${k} is required`; return v; };
+        switch (op) {
+            case 'http.fetch': return this.fetch(args, deadline);
+            case 'store.get': return this.storeGet(str('key')) ?? null;
+            case 'store.set':
+            case 'store.delete': {
+                const key = str('key');
+                if (key.length === 0 || key.length > 200)
+                    throw 'store keys are 1 to 200 characters';
+                if (op === 'store.delete') {
+                    this.storeDelete(key);
+                    return null;
+                }
+                const value = args.value ?? null;
+                if (JSON.stringify(value).length > MAX_KV_VALUE)
+                    throw 'store values are at most 64 KB';
+                if (args && 'if_equals' in args) {
+                    if (JSON.stringify(this.storeGet(key) ?? null) !== JSON.stringify(args.if_equals ?? null))
+                        return false;
+                    this.storeSet(key, value);
+                    return true;
+                }
+                this.storeSet(key, value);
+                return null;
+            }
+            case 'store.incr': {
+                const key = str('key');
+                if (key.length === 0 || key.length > 200)
+                    throw 'store keys are 1 to 200 characters';
+                const by = typeof args?.by === 'number' ? args.by : 1;
+                const cur = this.storeGet(key);
+                const n = (typeof cur === 'number' ? cur : 0) + by;
+                this.storeSet(key, n);
+                return n;
+            }
+            case 'crypto.hmac':
+            case 'crypto.hash': return cryptoOp(op, args, (n) => this.secret(n));
+            case 'records.upsert': {
+                const ok = (s) => this.m.permissions.scopes.includes(s) && this.granted.includes(s);
+                if (!ok('records.write'))
+                    throw 'not allowed: this connection was not granted records.write';
+                const r = args?.records;
+                return this.upsert(Array.isArray(r) ? r : [r]);
+            }
+            case 'knowledge.upsert':
+            case 'knowledge.delete': {
+                const ok = (s) => this.m.permissions.scopes.includes(s) && this.granted.includes(s);
+                if (!ok('knowledge.write'))
+                    throw 'not allowed: this connection was not granted knowledge.write';
+                if (op === 'knowledge.delete')
+                    return this.knowledge('delete', [String(args?.external_id ?? '')]);
+                const r = args?.records;
+                const records = Array.isArray(r) ? r : [r];
+                if (records.length > 500)
+                    throw 'at most 500 records per call';
+                return this.knowledge('upsert', records);
+            }
+            case 'log': {
+                const level = typeof args?.level === 'string' ? args.level : 'info';
+                this.log(level, String(args?.message ?? '').slice(0, 2000));
+                return null;
+            }
+            default: throw `unknown host call ${op}`;
+        }
+    }
+}
+const ALGS = { sha256: 'sha256', sha512: 'sha512' };
+/** crypto.rs fill_secrets: each `{{secrets.x}}` becomes that declared secret. */
+export function fillSecrets(text, secret) {
+    let out = '';
+    let rest = text;
+    for (let start = rest.indexOf('{{secrets.'); start >= 0; start = rest.indexOf('{{secrets.')) {
+        const after = rest.slice(start + 10);
+        const end = after.indexOf('}}');
+        if (end < 0)
+            throw 'unclosed {{secrets.';
+        const name = after.slice(0, end).trim();
+        const value = secret(name);
+        if (value === undefined)
+            throw `the install is missing secret ${JSON.stringify(name)}; reconnect it`;
+        out += rest.slice(0, start) + value;
+        rest = after.slice(end + 2);
+    }
+    return out + rest;
+}
+/** crypto.rs host_op: `{{secrets.x}}` in the message is filled in, keys never leave the host. */
+export function cryptoOp(op, args, secret) {
+    const get = (k) => typeof args?.[k] === 'string' ? args[k] : undefined;
+    const algName = (get('alg') ?? 'sha256').toLowerCase().replaceAll('-', '');
+    const raw = get('message');
+    if (raw === undefined)
+        throw `${op}: message is required`;
+    // Filled only for an HMAC: a plain hash would let code guess the secret.
+    const message = op === 'crypto.hmac' ? fillSecrets(raw, secret) : raw;
+    const alg = ALGS[algName];
+    let bytes;
+    if (op === 'crypto.hmac') {
+        const name = get('secret');
+        if (name === undefined)
+            throw "crypto.hmac: secret (the secret's name) is required";
+        const key = secret(name);
+        if (!key)
+            throw `the install is missing secret ${JSON.stringify(name)}; reconnect it`;
+        if (!alg)
+            throw `unknown HMAC algorithm ${JSON.stringify(algName)}; use sha256 or sha512`;
+        bytes = createHmac(alg, key).update(message).digest();
+    }
+    else {
+        if (!alg)
+            throw `unknown hash ${JSON.stringify(algName)}; use sha256 or sha512`;
+        bytes = createHash(alg).update(message).digest();
+    }
+    const encoding = get('encoding') ?? 'hex';
+    if (encoding !== 'hex' && encoding !== 'base64')
+        throw `${op}: encoding ${JSON.stringify(encoding)} is hex or base64`;
+    return bytes.toString(encoding);
+}
+export class LocalHost extends BaseHost {
+    secrets;
+    storePath;
+    egress;
+    out;
+    store;
+    constructor(m, config, secrets, storePath, egress, out) {
+        super(m, config, m.permissions.scopes);
+        this.secrets = secrets;
+        this.storePath = storePath;
+        this.egress = egress;
+        this.out = out;
+        this.store = existsSync(storePath) ? JSON.parse(readFileSync(storePath, 'utf8')) : {};
+    }
+    save() { mkdirSync(dirname(this.storePath), { recursive: true }); writeFileSync(this.storePath, JSON.stringify(this.store, null, 2)); }
+    storeGet(key) { return this.store[key]; }
+    storeSet(key, value) { this.store[key] = value; this.save(); }
+    storeDelete(key) { delete this.store[key]; this.save(); }
+    fetch(args, deadline) {
+        const method = String(args?.method ?? 'GET').toUpperCase();
+        // The host owns auth: JS cannot set or override it.
+        const headers = pairs(args?.headers).filter(([k]) => k.toLowerCase() !== 'authorization');
+        const query = pairs(args?.query);
+        try {
+            applyAuth(this.m.auth, this.secrets, headers, query);
+        }
+        catch (e) {
+            throw e.message;
+        }
+        const req = { method, url: String(args?.url), headers, query, body: typeof args?.body === 'string'
+                // Declared secrets named as {{secrets.x}} are filled in by the host, as on the server.
+                ? (args.body.includes('{{secrets.') ? fillSecrets(args.body, (n) => this.secret(n)) : args.body)
+                : null };
+        const t = Date.now();
+        const resp = fetchSync(allowedHosts(this.m, this.config), req, this.egress, deadline);
+        this.out(`http ${method} ${req.url} -> ${resp.status} (${Date.now() - t} ms)`);
+        return { status: resp.status, headers: resp.headers, body: resp.body };
+    }
+    upsert(records) {
+        this.out(`records.upsert ${JSON.stringify(records)}`);
+        return { local: true, count: records.length };
+    }
+    knowledge(kind, items) {
+        this.out(`knowledge.${kind} ${JSON.stringify(items)}`);
+        return kind === 'delete' ? { local: true, deleted: true } : { local: true, results: items.map(() => ({ result: 'written' })) };
+    }
+    secret(name) {
+        if (!secretNames(this.m).includes(name))
+            return undefined;
+        const v = this.secrets[name];
+        return typeof v === 'string' ? v : undefined;
+    }
+    log(level, message) {
+        this.out(this.m.sensitive ? `log ${level} (sensitive: withheld on AirFone) ${message}` : `log ${level} ${message}`);
+    }
+}
+export class ReviewHost extends BaseHost {
+    store = new Map();
+    constructor(m, config) { super(m, config, m.permissions.scopes); }
+    storeGet(key) { return this.store.get(key); }
+    storeSet(key, value) { this.store.set(key, value); }
+    storeDelete(key) { this.store.delete(key); }
+    fetch() { throw 'no network during review; give fixtures that need none'; }
+    upsert() { return { review: true }; }
+    knowledge() { return { review: true }; }
+    log() { }
+    secret() { return 'review'; }
+    call(op, args, deadline) {
+        // The server's review host does not check scopes for records.
+        if (op === 'records.upsert' || op === 'knowledge.upsert' || op === 'knowledge.delete')
+            return { review: true };
+        return super.call(op, args, deadline);
+    }
+}

package/dist/http.d.ts

@@ -0,0 +1,34 @@
+import { type Canonical } from './manifest.js';
+import { type EgressOptions, type OutRequest } from './net.js';
+type Secrets = Record<string, string>;
+/** `secrets` is readable as `{{secrets.x}}` (headers and query only); `keys` feeds the hmac filters anywhere. */
+interface Scope {
+    input: unknown;
+    config: unknown;
+    secrets: Secrets | null;
+    keys?: Secrets | null;
+}
+export declare const encodeComponent: (s: string) => string;
+/** `a ?? b | filter | filter` (template.rs Scope::evaluate). Undefined when every reference is absent. */
+export declare function evaluate(expr: string, scope: Scope): unknown;
+export declare const renderStr: (s: string, scope: Scope) => string;
+export declare const renderUrl: (s: string, scope: Scope) => string;
+export declare function renderValue(v: unknown, scope: Scope): unknown;
+/** Adds the manifest's auth to a request (http_exec.rs apply_auth). */
+export declare function applyAuth(auth: Canonical['auth'], secrets: Secrets, headers: [string, string][], query: [string, string][]): void;
+export declare function buildRequest(m: Canonical, spec: any, input: unknown, config: unknown, secrets: Secrets): OutRequest;
+export declare function mapOutput(map: Record<string, string> | undefined, body: unknown): unknown;
+/** The answer as JSON (http_exec.rs response_json). An event stream gives its last JSON data event. */
+export declare function responseJson(resp: {
+    headers: Record<string, string> | [string, string][];
+    body: string;
+}): unknown;
+/** http_exec.rs check_fail_rules. */
+export declare function checkFailRules(rules: {
+    path: string;
+    equals?: unknown;
+    message?: string;
+    message_path?: string;
+}[] | undefined, body: unknown): void;
+export declare function runHttp(m: Canonical, action: any, input: unknown, config: unknown, secrets: Secrets, egress: EgressOptions): Promise<unknown>;
+export {};

package/dist/http.js

@@ -0,0 +1,268 @@
+// Declarative HTTP actions, as the server runs them (template.rs, http_exec.rs).
+import { createHash, createHmac } from 'node:crypto';
+import { JSONPath } from 'jsonpath-plus';
+import { allowedHosts, withConfigDefaults } from './manifest.js';
+import { send } from './net.js';
+const refs = (s) => [...s.matchAll(/\{\{(.*?)\}\}/g)].map((m) => m[1].trim());
+function lookup(root, path) {
+    let cur = root;
+    for (const p of path) {
+        if (Array.isArray(cur)) {
+            if (!/^\d+$/.test(p))
+                return undefined;
+            cur = cur[Number(p)];
+        }
+        else if (cur && typeof cur === 'object')
+            cur = cur[p];
+        else
+            return undefined;
+        if (cur === undefined)
+            return undefined;
+    }
+    return cur;
+}
+function resolve(ref, scope) {
+    const [head, ...rest] = ref.split('.');
+    if (head === 'secrets') {
+        if (!scope.secrets)
+            throw new Error('secrets can only be used in headers and query parameters');
+        if (rest.length !== 1)
+            throw new Error(`bad secret reference ${JSON.stringify(ref)}`);
+        return scope.secrets[rest[0]];
+    }
+    if (head === 'input')
+        return lookup(scope.input, rest);
+    if (head === 'config')
+        return lookup(scope.config, rest);
+    throw new Error(`unknown template root in {{${ref}}}`);
+}
+const asText = (v) => v === undefined || v === null ? '' : typeof v === 'string' ? v : JSON.stringify(v);
+export const encodeComponent = (s) => [...new TextEncoder().encode(s)].map((b) => /[A-Za-z0-9\-_.~]/.test(String.fromCharCode(b)) ? String.fromCharCode(b) : `%${b.toString(16).toUpperCase().padStart(2, '0')}`).join('');
+/** `a ?? b | filter | filter` (template.rs Scope::evaluate). Undefined when every reference is absent. */
+export function evaluate(expr, scope) {
+    const [refsPart, ...filters] = expr.split('|');
+    let value;
+    for (const r of refsPart.split('??').map((x) => x.trim())) {
+        const v = resolve(r, scope);
+        if (v === undefined || v === null || v === '')
+            continue;
+        value = v;
+        break;
+    }
+    for (const f of filters.map((x) => x.trim())) {
+        if (value === undefined)
+            break;
+        value = filter(f, asText(value), scope);
+    }
+    return value;
+}
+function filter(f, text, scope) {
+    const open = f.indexOf('(');
+    const name = open < 0 ? f : f.slice(0, open).trim();
+    const arg = open < 0 ? null : f.slice(open + 1).replace(/\)+$/, '').trim().replace(/^["']+|["']+$/g, '');
+    const key = () => {
+        if (arg === null)
+            throw new Error(`${name} needs a secret name`);
+        const k = scope.keys?.[arg];
+        if (typeof k !== 'string')
+            throw new Error(`the install is missing secret ${JSON.stringify(arg)}; reconnect it`);
+        return k;
+    };
+    switch (name) {
+        case 'base64': return Buffer.from(text).toString('base64');
+        case 'urlencode': return encodeComponent(text);
+        case 'sha256': return createHash('sha256').update(text).digest('hex');
+        case 'hmac_sha256': return createHmac('sha256', key()).update(text).digest('hex');
+        case 'hmac_sha256_b64': return createHmac('sha256', key()).update(text).digest('base64');
+        default: throw new Error(`unknown template filter ${JSON.stringify(name)}`);
+    }
+}
+function renderWith(s, scope, encode) {
+    let out = '';
+    let rest = s;
+    for (;;) {
+        const start = rest.indexOf('{{');
+        if (start < 0)
+            break;
+        out += rest.slice(0, start);
+        const after = rest.slice(start + 2);
+        const end = after.indexOf('}}');
+        if (end < 0)
+            throw new Error(`unclosed {{ in template ${JSON.stringify(s)}`);
+        out += encode(out.length, asText(evaluate(after.slice(0, end).trim(), scope)));
+        rest = after.slice(end + 2);
+    }
+    return out + rest;
+}
+export const renderStr = (s, scope) => renderWith(s, scope, (_, t) => t);
+export const renderUrl = (s, scope) => renderWith(s, scope, (at, t) => at === 0 ? t.replace(/\/+$/, '') : encodeComponent(t));
+export function renderValue(v, scope) {
+    if (typeof v === 'string') {
+        const t = v.trim();
+        const r = refs(t);
+        if (isSingleReference(v) && r.length === 1)
+            return evaluate(r[0], scope) ?? null;
+        return renderStr(v, scope);
+    }
+    if (Array.isArray(v))
+        return v.map((x) => renderValue(x, scope));
+    if (v && typeof v === 'object') {
+        const out = {};
+        for (const [k, x] of Object.entries(v)) {
+            const r = renderValue(x, scope);
+            // A field that is only one reference, and that reference is absent, is left out.
+            if (r === null && isSingleReference(x))
+                continue;
+            out[k] = r;
+        }
+        return out;
+    }
+    return v;
+}
+function isSingleReference(v) {
+    if (typeof v !== 'string')
+        return false;
+    const t = v.trim();
+    return t.startsWith('{{') && t.endsWith('}}') && t.split('{{').length === 2;
+}
+function secret(secrets, name) {
+    const v = secrets[name];
+    if (typeof v !== 'string' || !v)
+        throw new Error(`the install is missing secret ${JSON.stringify(name)}; reconnect it`);
+    return v;
+}
+/** Adds the manifest's auth to a request (http_exec.rs apply_auth). */
+export function applyAuth(auth, secrets, headers, query) {
+    const a = auth;
+    switch (a.kind) {
+        case 'api_key': {
+            const value = `${a.prefix ?? ''}${secret(secrets, a.secret)}`;
+            (a.in === 'query' ? query : headers).push([a.name, value]);
+            break;
+        }
+        case 'bearer':
+            headers.push(['Authorization', `Bearer ${secret(secrets, a.secret)}`]);
+            break;
+        case 'oauth':
+            headers.push(['Authorization', `${typeof a.prefix === 'string' ? a.prefix : 'Bearer '}${secret(secrets, a.secret)}`]);
+            break;
+        case 'basic':
+            headers.push(['Authorization', `Basic ${Buffer.from(`${secret(secrets, a.username)}:${secret(secrets, a.password)}`).toString('base64')}`]);
+            break;
+    }
+}
+export function buildRequest(m, spec, input, config, secrets) {
+    const plain = { input, config, secrets: null, keys: secrets };
+    const withSecrets = { input, config, secrets, keys: secrets };
+    const url = renderUrl(spec.url, plain);
+    const headers = Object.entries(spec.headers ?? {}).map(([k, v]) => [k, renderStr(String(v), withSecrets)]);
+    const query = [];
+    for (const [k, v] of Object.entries(spec.query ?? {})) {
+        const value = renderStr(String(v), withSecrets);
+        if (value)
+            query.push([k, value]);
+    }
+    applyAuth(m.auth, secrets, headers, query);
+    let body = null;
+    if (spec.body !== undefined && spec.body !== null) {
+        if (!headers.some(([k]) => k.toLowerCase() === 'content-type'))
+            headers.push(['Content-Type', 'application/json']);
+        body = JSON.stringify(renderValue(spec.body, plain));
+    }
+    return { method: String(spec.method).toUpperCase(), url, headers, query, body };
+}
+export function mapOutput(map, body) {
+    if (!map || Object.keys(map).length === 0)
+        return body;
+    const out = {};
+    for (const [field, path] of Object.entries(map)) {
+        let nodes;
+        try {
+            nodes = JSONPath({ path, json: body, wrap: true });
+        }
+        catch (e) {
+            throw new Error(`output ${field}: bad JSONPath ${JSON.stringify(path)}: ${e.message}`);
+        }
+        const many = path.includes('[*]') || path.includes('..') || path.includes('[?');
+        out[field] = many ? nodes : nodes[0] ?? null;
+    }
+    return out;
+}
+const short = (s) => s.slice(0, 200).replaceAll('\n', ' ');
+/** The answer as JSON (http_exec.rs response_json). An event stream gives its last JSON data event. */
+export function responseJson(resp) {
+    if (!resp.body.trim())
+        return null;
+    const headers = Array.isArray(resp.headers) ? resp.headers : Object.entries(resp.headers);
+    const start = resp.body.trimStart();
+    const sse = headers.some(([k, v]) => k.toLowerCase() === 'content-type' && String(v).startsWith('text/event-stream')) || start.startsWith('data:') || start.startsWith('event:');
+    if (sse) {
+        const last = lastSseJson(resp.body);
+        if (last === undefined)
+            throw new Error(`the event stream has no JSON event: ${short(resp.body)}`);
+        return last;
+    }
+    try {
+        return JSON.parse(resp.body);
+    }
+    catch (e) {
+        throw new Error(`the answer is not JSON (${e.message}): ${short(resp.body)}`);
+    }
+}
+function lastSseJson(body) {
+    let data = '';
+    let last;
+    for (const line of [...body.split(/\r?\n/), '']) {
+        if (line.startsWith('data:'))
+            data += line.slice(5).trimStart();
+        else if (line === '' && data) {
+            try {
+                last = JSON.parse(data);
+            }
+            catch { /* not JSON */ }
+            data = '';
+        }
+    }
+    return last;
+}
+/** http_exec.rs check_fail_rules. */
+export function checkFailRules(rules, body) {
+    const first = (path) => {
+        try {
+            const r = JSONPath({ path, json: body, wrap: true });
+            return r.length ? r[0] : undefined;
+        }
+        catch (e) {
+            throw new Error(`fail_when: ${e.message}`);
+        }
+    };
+    for (const rule of rules ?? []) {
+        const hit = first(rule.path);
+        const failed = rule.equals !== undefined && rule.equals !== null ? hit !== undefined && JSON.stringify(hit) === JSON.stringify(rule.equals) : hit !== undefined && hit !== null;
+        if (!failed)
+            continue;
+        let said;
+        if (rule.message_path) {
+            try {
+                const v = first(rule.message_path);
+                if (v !== undefined)
+                    said = typeof v === 'string' ? v : JSON.stringify(v);
+            }
+            catch { /* ignored */ }
+        }
+        throw new Error(rule.message ?? said ?? 'the service refused the request');
+    }
+}
+export async function runHttp(m, action, input, config, secrets, egress) {
+    config = withConfigDefaults(m.config, config);
+    const req = buildRequest(m, action.http, input, config, secrets);
+    const resp = await send(allowedHosts(m, config), req, egress);
+    const msg = action.http.errors?.[String(resp.status)];
+    if (msg)
+        throw new Error(msg);
+    if (resp.status < 200 || resp.status >= 300)
+        throw new Error(`HTTP ${resp.status}: ${short(resp.body)}`);
+    const body = responseJson(resp);
+    checkFailRules(action.http.fail_when, body);
+    return mapOutput(action.http.output, body);
+}

package/dist/index.d.ts

@@ -0,0 +1,8 @@
+export type * from './types.js';
+import type { Actions } from './types.js';
+/** Types your actions object. Returns it unchanged: `export const actions = defineActions({ ... })`. */
+export declare function defineActions<A extends Actions>(actions: A): A;
+export { parseManifest, normalize, checkSchema, permissionDiff, ManifestError, SCOPES } from './manifest.js';
+export { runAction, compileCheck, DEFAULT_LIMITS, ScriptError, type HostBridge, type Limits } from './runner.js';
+export { pack, sha256Hex, signingMessage, signBundle } from './bundle.js';
+export { review, type Finding } from './review.js';

package/dist/index.js

@@ -0,0 +1,8 @@
+/** Types your actions object. Returns it unchanged: `export const actions = defineActions({ ... })`. */
+export function defineActions(actions) {
+    return actions;
+}
+export { parseManifest, normalize, checkSchema, permissionDiff, ManifestError, SCOPES } from './manifest.js';
+export { runAction, compileCheck, DEFAULT_LIMITS, ScriptError } from './runner.js';
+export { pack, sha256Hex, signingMessage, signBundle } from './bundle.js';
+export { review } from './review.js';

package/dist/knowledge.d.ts

@@ -0,0 +1,6 @@
+export declare const KNOWLEDGE_HELP = "airfone knowledge <command>\n\nCommands\n  push <source> <file>       Upsert the records in a JSON array or JSON Lines file\n                             into <source> (a-z, 0-9, - and _). 500 per request.\n  delete <source> <id>       Delete one record by its external_id.\n  sources                    List your sources with record counts.\n\nEach record: {\"external_id\", \"kind\", \"name\", \"aliases\"?, \"facts\": [{\"relation\",\n\"text\", \"audience\"?, \"volatile\"?, \"valid_from\"?, \"valid_to\"?}], \"links\"?, \"updated_at\"?}\n\nNeeds AIRFONE_KEY (a secret API key with knowledge:write) or AIRFONE_TOKEN, and\nAIRFONE_API (default https://app.airfone.app).";
+type Out = (s: string) => void;
+/** Records from a JSON array file or a JSON Lines file. */
+export declare function readRecords(path: string): unknown[];
+export declare function knowledge(pos: string[], out: Out, err: Out): Promise<boolean>;
+export {};