@graphty/graph-io 0.3.47 → 0.3.49

package/dist/src/sources/run.js

@@ -0,0 +1,673 @@
+/**
+ * Running a source (design sections 2, 3 and 5): the checks before any request, the three read
+ * paths (`records` through the record methods, `file` through the format importers, `import`
+ * straight into the sink), the caps between pages, `idPrefix`, the versioned cursor, and the
+ * report's `source` member. graph-io keeps no state between calls.
+ */
+import { INVALID_INDEX } from "@graphty/graph-format";
+import { MISSING_ID_CODE } from "../common/codes.js";
+import { readBytes, throwIfAborted } from "../common/input.js";
+import { ImportReportBuilder } from "../common/report.js";
+import { readZipDirectory, readZipEntry } from "../common/zip.js";
+import { SNIFF_HEAD_BYTES } from "../sniff.js";
+import { ImportError } from "../types.js";
+import { createSourceContext, detectRuntime, LimitStop } from "./context.js";
+import { originOf } from "./credentials.js";
+import { checkParams } from "./define.js";
+import { failOn, warnOn } from "./fail.js";
+import { SOURCE_FORMAT_PREFIX } from "./ids.js";
+/** The cursor key graph-io adds, holding the source's descriptor version. */
+const CURSOR_VERSION_KEY = "~v";
+/** The options that belong to the source functions, never to an importer. */
+const SOURCE_OPTION_KEYS = new Set([
+    "endpoint",
+    "credential",
+    "cursor",
+    "limits",
+    "retry",
+    "idPrefix",
+    "allowInsecure",
+    "fetch",
+    "userAgent",
+    "builder",
+    "freeze",
+    "onProgress",
+    "signal",
+]);
+/** The statuses whose failure on a later page keeps the pages before it. */
+const AUTH_CODES = new Set(["E_SOURCE_AUTH", "E_SOURCE_FORBIDDEN"]);
+/**
+ * The report format name of a source load.
+ * @param id - the source id
+ * @returns `"source:<id>"`
+ */
+export function sourceFormat(id) {
+    return `${SOURCE_FORMAT_PREFIX}${id}`;
+}
+/**
+ * The cursor handed back to the caller: the source's own, with its version added.
+ * @param next - the source's cursor
+ * @param version - the descriptor version
+ * @returns the caller's cursor
+ */
+export function wrapCursor(next, version) {
+    return next === null || next === undefined ? null : Object.freeze({ ...next, [CURSOR_VERSION_KEY]: version });
+}
+/**
+ * Check every rule that holds before a request is sent and build the context.
+ * @param source - the loaded source
+ * @param rawParams - the caller's params
+ * @param options - the caller's options
+ * @param unit - what progress counts
+ * @returns the report, context and params
+ */
+function prepare(source, rawParams, options, unit) {
+    const { descriptor } = source;
+    const version = descriptor?.version ?? 1;
+    const report = new ImportReportBuilder(sourceFormat(source.id), options.errorLimit ?? 100);
+    checkRuntime(source, report);
+    checkParams(rawParams, descriptor, report);
+    const params = {};
+    for (const p of descriptor?.params ?? []) {
+        if (p.default !== undefined) {
+            params[p.name] = p.default;
+        }
+    }
+    Object.assign(params, rawParams);
+    const endpoint = resolveEndpoint(options.endpoint ?? descriptor?.defaultEndpoint ?? null, report);
+    if (options.credential?.origins.length === 0 && endpoint === null) {
+        failOn(report, "E_SOURCE_PARAM", "a credential needs an endpoint or its own origins", {
+            name: "credential",
+            reason: "credential-unbound",
+        });
+    }
+    const cursor = resumeCursor(options.cursor, version, report);
+    const counts = { nodes: new Set(), edgesAtStart: 0, sink: null };
+    const created = createSourceContext({
+        id: source.id,
+        descriptor,
+        options,
+        report,
+        endpoint,
+        cursor,
+        unit,
+        counts: () => ({
+            nodes: counts.nodes.size,
+            edges: counts.sink === null ? 0 : counts.sink.edgeCount - counts.edgesAtStart,
+        }),
+    });
+    return { report, created, params, counts };
+}
+/**
+ * Refuse a source that cannot run here: a runtime it does not support, or a secure context it needs.
+ * @param source - the loaded source
+ * @param report - the load's report
+ */
+function checkRuntime(source, report) {
+    const { descriptor } = source;
+    const runtime = detectRuntime();
+    const support = descriptor?.runtimes?.[runtime];
+    if (support?.supported === false) {
+        failOn(report, "E_SOURCE_RUNTIME", `source ${source.id} cannot run in ${runtime}`, {
+            runtime,
+            reason: support.reason ?? "unsupported",
+        });
+    }
+    if (descriptor?.needs?.secureContext === true &&
+        runtime === "browser" &&
+        globalThis.isSecureContext === false) {
+        failOn(report, "E_SOURCE_RUNTIME", `source ${source.id} needs a secure context`, {
+            runtime,
+            reason: "needs-secure-context",
+        });
+    }
+}
+/**
+ * The endpoint as a URL, refusing one that is not a URL or carries a user name or password.
+ * @param given - the endpoint given, if any
+ * @param report - the load's report
+ * @returns the URL, or null without one
+ */
+function resolveEndpoint(given, report) {
+    if (given === null) {
+        return null;
+    }
+    let endpoint;
+    try {
+        endpoint = new URL(given);
+    }
+    catch {
+        failOn(report, "E_SOURCE_PARAM", "the endpoint is not a URL", { name: "endpoint", reason: "type" });
+    }
+    if (endpoint.username !== "" || endpoint.password !== "") {
+        failOn(report, "E_SOURCE_PARAM", "the endpoint may not carry a user name or password", {
+            name: "endpoint",
+            reason: "credentials-in-url",
+        });
+    }
+    return endpoint;
+}
+/**
+ * The source's own cursor from the caller's, refusing one another descriptor version made.
+ * @param given - the caller's cursor, if any
+ * @param version - the descriptor version
+ * @param report - the load's report
+ * @returns the source's cursor, or null
+ */
+function resumeCursor(given, version, report) {
+    if (given === undefined || given === null) {
+        return null;
+    }
+    const { [CURSOR_VERSION_KEY]: recorded, ...rest } = given;
+    if (recorded !== undefined && recorded !== version) {
+        failOn(report, "E_SOURCE_PARAM", "the cursor was made by another version of the source", {
+            name: "cursor",
+            reason: "stale-cursor",
+        });
+    }
+    return Object.freeze(rest);
+}
+/**
+ * Wrap a sink so every node id gets the prefix and the caps can count the distinct nodes written.
+ * @param sink - the caller's sink or the builder
+ * @param prepared - the call's counters
+ * @param prefix - the idPrefix, if any
+ * @returns the wrapped sink
+ */
+export function wrapSink(sink, prepared, prefix) {
+    prepared.counts.sink = sink;
+    prepared.counts.edgesAtStart = sink.edgeCount;
+    const seen = prepared.counts.nodes;
+    const id = (value) => (prefix === undefined || prefix === "" ? value : `${prefix}${String(value)}`);
+    const node = (index) => {
+        seen.add(index);
+        return index;
+    };
+    // an edge may add its end nodes (addMissingNodes)
+    const ends = (edge) => {
+        const [a, b] = sink.edgeEndpoints(edge);
+        seen.add(a);
+        seen.add(b);
+        return edge;
+    };
+    return new Proxy(sink, {
+        get(target, key) {
+            switch (key) {
+                case "addNode":
+                    return (v) => node(target.addNode(id(v)));
+                case "addNodeRecord":
+                    return (v, attrs) => node(target.addNodeRecord(id(v), attrs));
+                case "addNodes":
+                    return (ids, out) => {
+                        const result = target.addNodes(Array.from(ids, id), out);
+                        result.forEach((i) => seen.add(i));
+                        return result;
+                    };
+                case "addEdge":
+                    return (s, t, w) => ends(target.addEdge(id(s), id(t), w));
+                case "addEdgeRecord":
+                    return (s, t, attrs, weightKey) => ends(target.addEdgeRecord(id(s), id(t), attrs, weightKey));
+                case "indexOf":
+                    return (v) => target.indexOf(id(v));
+                default: {
+                    const value = Reflect.get(target, key, target);
+                    return typeof value === "function" ? value.bind(target) : value;
+                }
+            }
+        },
+    });
+}
+/**
+ * The options a format importer gets from a file source's load.
+ * @param options - the caller's options
+ * @param signal - the context's signal
+ * @returns the importer options
+ */
+function importerOptions(options, signal) {
+    const out = { signal };
+    for (const [key, value] of Object.entries(options)) {
+        if (!SOURCE_OPTION_KEYS.has(key)) {
+            out[key] = value;
+        }
+    }
+    return out;
+}
+/**
+ * Write graph facts: `name`, `description` and `modified` strings to the graph's meta, any other
+ * scalar (but `directed`) as a graph value.
+ * @param sink - the sink
+ * @param meta - the facts
+ */
+function writeMeta(sink, meta) {
+    const patch = {};
+    for (const [key, value] of Object.entries(meta)) {
+        if ((key === "name" || key === "description" || key === "modified") && typeof value === "string") {
+            patch[key] = value;
+        }
+        else if (key !== "directed" &&
+            (typeof value === "string" || typeof value === "number" || typeof value === "boolean")) {
+            sink.setGraphValue(key, value);
+        }
+    }
+    if (Object.keys(patch).length > 0) {
+        sink.setMeta(patch);
+    }
+}
+/**
+ * Write one page of records into the sink.
+ * @param batch - the page
+ * @param sink - the wrapped sink
+ * @param source - the source (for its record keys)
+ * @param prepared - the call
+ * @param first - whether this is the first page
+ */
+function writeBatch(batch, sink, source, prepared, first) {
+    const idKey = source.idKey ?? "id";
+    const sourceKey = source.sourceKey ?? "source";
+    const targetKey = source.targetKey ?? "target";
+    const meta = batch.meta ?? {};
+    if (first && typeof meta.directed === "boolean" && sink.edgeCount === 0) {
+        sink.setDirected(meta.directed);
+    }
+    writeMeta(sink, meta);
+    const n = writeNodes(batch, sink, idKey, prepared);
+    writeEdges(batch, sink, sourceKey, targetKey, prepared, n);
+}
+/**
+ * Write a page's node records.
+ * @param batch - the page
+ * @param sink - the wrapped sink
+ * @param idKey - the record key holding the id
+ * @param prepared - the call
+ * @returns how many records were visited, for the abort check's cadence
+ */
+function writeNodes(batch, sink, idKey, prepared) {
+    const { report } = prepared;
+    const { signal } = prepared.created.ctx;
+    let n = 0;
+    for (const record of batch.nodes ?? []) {
+        if (n++ % 64 === 0) {
+            throwIfAborted(signal);
+        }
+        const { [idKey]: id, ...attrs } = record;
+        if (typeof id !== "string" && typeof id !== "number") {
+            report.error("missing-value", MISSING_ID_CODE, `a node record has no ${idKey}`);
+            report.counts.skippedNodes++;
+            continue;
+        }
+        try {
+            sink.addNodeRecord(id, attrs);
+            report.counts.nodes++;
+        }
+        catch (err) {
+            report.recordError(err, { element: String(id) });
+            report.counts.skippedNodes++;
+        }
+    }
+    return n;
+}
+/**
+ * Write a page's edge records.
+ * @param batch - the page
+ * @param sink - the wrapped sink
+ * @param sourceKey - the record key holding the source id
+ * @param targetKey - the record key holding the target id
+ * @param prepared - the call
+ * @param start - records visited so far, for the abort check's cadence
+ */
+function writeEdges(batch, sink, sourceKey, targetKey, prepared, start) {
+    const { report } = prepared;
+    const { signal } = prepared.created.ctx;
+    let n = start;
+    let edgeIds = null;
+    for (const record of batch.edges ?? []) {
+        if (n++ % 64 === 0) {
+            throwIfAborted(signal);
+        }
+        const s = record[sourceKey];
+        const t = record[targetKey];
+        const edgeId = record.id;
+        const fields = Object.fromEntries(Object.entries(record).filter(([k]) => k !== sourceKey && k !== targetKey && k !== "id"));
+        if ((typeof s !== "string" && typeof s !== "number") || (typeof t !== "string" && typeof t !== "number")) {
+            report.error("missing-value", MISSING_ID_CODE, `an edge record has no ${sourceKey} or ${targetKey}`);
+            report.counts.skippedEdges++;
+            continue;
+        }
+        try {
+            const e = sink.addEdgeRecord(s, t, fields);
+            if (typeof edgeId === "string" || typeof edgeId === "number") {
+                edgeIds ?? (edgeIds = edgeIdColumn(sink));
+                sink.setEdgeValue(edgeIds, e, String(edgeId));
+            }
+            report.counts.edges++;
+        }
+        catch (err) {
+            report.recordError(err, { element: `${String(s)} -> ${String(t)}` });
+            report.counts.skippedEdges++;
+        }
+    }
+}
+/**
+ * The origin of an endpoint, or "" without one.
+ * @param endpoint - the endpoint
+ * @returns its origin
+ */
+function endpointOrigin(endpoint) {
+    return endpoint === null ? "" : originOf(endpoint);
+}
+/**
+ * The edge id column, declared on first use.
+ * @param sink - the sink
+ * @returns its handle
+ */
+function edgeIdColumn(sink) {
+    const existing = sink.edgeColumn("id");
+    return existing === INVALID_INDEX
+        ? sink.declareEdgeColumn({ name: "id", dtype: "string", role: "id", nullable: true })
+        : existing;
+}
+/**
+ * The page iterator of a records source: its async iterable, or its one batch.
+ * @param out - what `records()` returned
+ * @returns an iterator over the pages
+ */
+function pages(out) {
+    if (Symbol.asyncIterator in out) {
+        return out[Symbol.asyncIterator]();
+    }
+    let done = false;
+    return {
+        async next() {
+            if (done) {
+                return { done: true, value: undefined };
+            }
+            done = true;
+            return { done: false, value: await out };
+        },
+    };
+}
+/**
+ * Decide what a failed page means: a cap reached or a later page needing a credential stops the
+ * load and keeps what was read (warning for the credential); anything else is rethrown.
+ * @param err - what the page threw
+ * @param page - how many pages were written before it
+ * @param prepared - the call
+ */
+function stopOrThrow(err, page, prepared) {
+    if (err instanceof LimitStop) {
+        return;
+    }
+    if (page > 0 && err instanceof ImportError && AUTH_CODES.has(String(err.issue?.code))) {
+        const p = err.issue?.params ?? {};
+        warnOn(prepared.report, "W_SOURCE_AUTH", "a later page needs a credential; the pages before it are kept", {
+            origin: typeof p.origin === "string" ? p.origin : endpointOrigin(prepared.created.ctx.endpoint),
+            status: typeof p.status === "number" ? p.status : null,
+        });
+        return;
+    }
+    throw err;
+}
+/**
+ * Run a records source into a sink.
+ * @param source - the source
+ * @param sink - the wrapped sink
+ * @param prepared - the call
+ * @returns where to resume, as the source gave it
+ */
+async function runRecords(source, sink, prepared) {
+    const { ctx } = prepared.created;
+    const records = source.records;
+    const iterator = pages(records(prepared.params, ctx));
+    let next = ctx.cursor;
+    let page = 0;
+    let stopped = false;
+    try {
+        for (;;) {
+            let step;
+            try {
+                step = await iterator.next();
+            }
+            catch (err) {
+                stopOrThrow(err, page, prepared);
+                stopped = true;
+                return next;
+            }
+            if (step.done === true) {
+                return page === 0 ? null : next;
+            }
+            writeBatch(step.value, sink, source, prepared, page === 0);
+            page++;
+            next = step.value.next ?? null;
+            ctx.progress(prepared.report.counts.nodes + prepared.report.counts.edges, undefined, "items");
+            if (ctx.limits.reached()) {
+                stopped = true;
+                return next;
+            }
+        }
+    }
+    finally {
+        if (stopped) {
+            await iterator.return?.().catch(() => undefined);
+        }
+    }
+}
+/**
+ * The importer input of a file source's body, gunzipped or unzipped when its name says so.
+ * @param file - what the source returned
+ * @param formats - the registry, for choosing a zip entry
+ * @param signal - the signal
+ * @returns the input and its hints
+ */
+async function fileInput(file, formats, signal) {
+    let input = file.body instanceof Response ? (file.body.body ?? new Uint8Array(0)) : file.body;
+    let { filename } = file;
+    let mimeType = file.mimeType ??
+        (file.body instanceof Response ? (file.body.headers.get("content-type") ?? undefined) : undefined);
+    const gzip = /\.gz$/i.test(filename ?? "") || /gzip/i.test(mimeType ?? "");
+    const zip = /\.zip$/i.test(filename ?? "") || /\/(x-)?zip\b/i.test(mimeType ?? "");
+    if (gzip) {
+        const stream = input instanceof ReadableStream
+            ? input
+            : new Response((await bytesOf(input, signal))).body;
+        input = (stream ?? new Response("").body ?? new ReadableStream()).pipeThrough(new DecompressionStream("gzip"));
+        filename = filename?.replace(/\.gz$/i, "");
+        mimeType = undefined;
+    }
+    else if (zip) {
+        const bytes = await bytesOf(input, signal);
+        const entry = readZipDirectory(bytes).find((e) => !e.directory && formats.sniff({ filename: e.name }) !== null);
+        if (entry === undefined) {
+            throw new ImportError("the zip holds no graph file", new ImportReportBuilder("zip", 0).finish());
+        }
+        input = await readZipEntry(bytes, entry, { signal, maxBytes: Number.MAX_SAFE_INTEGER, maxRatio: 1000 });
+        filename = entry.name;
+        mimeType = undefined;
+    }
+    return { input, filename, mimeType };
+}
+/**
+ * The bytes of an input.
+ * @param input - the input
+ * @param signal - the signal
+ * @returns the bytes
+ */
+async function bytesOf(input, signal) {
+    return (await readBytes(input, { signal })) ?? new TextEncoder().encode(typeof input === "string" ? input : "");
+}
+/**
+ * Run a file source into a sink through the format importers.
+ * @param source - the source
+ * @param sink - the wrapped sink
+ * @param prepared - the call
+ * @param formats - the registry
+ * @param options - the caller's options
+ * @returns the format the file was read as
+ */
+async function runFile(source, sink, prepared, formats, options) {
+    const { ctx } = prepared.created;
+    const read = source.file;
+    const file = await read(prepared.params, ctx);
+    const { input, filename, mimeType } = await fileInput(file, formats, ctx.signal);
+    let { format } = file;
+    let body = input;
+    if (format === undefined) {
+        const bytes = await bytesOf(input, ctx.signal);
+        format = formats.sniff({ filename, mimeType, head: bytes.subarray(0, SNIFF_HEAD_BYTES) })?.format;
+        body = bytes;
+        if (format === undefined) {
+            failOn(prepared.report, "E_SOURCE_RESPONSE", "the file is in no format graph-io reads", {
+                expected: "graph-file",
+                contentType: mimeType ?? null,
+            });
+        }
+    }
+    const importer = formats.importer(format);
+    prepared.report.include(await importer.import(body, sink, importerOptions(options, ctx.signal)));
+    writeMeta(sink, file.meta ?? {});
+    return format;
+}
+/**
+ * Run a source's read method into a sink.
+ * @param source - the loaded source
+ * @param params - the caller's params
+ * @param sink - where to write
+ * @param options - the caller's options
+ * @param formats - the registry, for file sources
+ * @returns the report and the format
+ */
+export async function runIntoSink(source, params, sink, options, formats) {
+    const prepared = prepare(source, params, options, typeof source.file === "function" ? "bytes" : "items");
+    const { ctx } = prepared.created;
+    const version = source.descriptor?.version ?? 1;
+    const wrapped = wrapSink(sink, prepared, options.idPrefix);
+    try {
+        if (typeof source.records === "function") {
+            const next = await runRecords(source, wrapped, prepared);
+            throwIfAborted(ctx.signal);
+            return {
+                report: withSource(prepared.report.finish(), prepared, wrapCursor(next, version)),
+                format: sourceFormat(source.id),
+            };
+        }
+        if (typeof source.file === "function") {
+            const format = await runFile(source, wrapped, prepared, formats, options);
+            return { report: withSource(prepared.report.finish(), prepared, null), format };
+        }
+        const run = source.import;
+        const report = await stopAtLimit(run(prepared.params, wrapped, ctx), prepared);
+        throwIfAborted(ctx.signal);
+        return {
+            report: withSource(report, prepared, wrapCursor(prepared.created.state.finish?.next, version)),
+            format: sourceFormat(source.id),
+        };
+    }
+    catch (err) {
+        throw settle(err, prepared);
+    }
+    finally {
+        prepared.created.dispose();
+    }
+}
+/**
+ * Run an optional method of a source (expand, probe, list, validate) with a context.
+ * @param source - the loaded source
+ * @param feature - the method name
+ * @param params - the params checked against the descriptor
+ * @param options - the caller's options
+ * @param call - calls the method with the context
+ * @returns what the method returned, and the call's state
+ */
+export async function runMethod(source, feature, params, options, call) {
+    if (typeof source[feature] !== "function") {
+        failOn(new ImportReportBuilder(sourceFormat(source.id), 0), "E_SOURCE_UNSUPPORTED", `source ${source.id} has no ${feature}`, {
+            feature,
+        });
+    }
+    const prepared = prepare(source, params, options, "items");
+    try {
+        const out = await call(prepared);
+        throwIfAborted(prepared.created.ctx.signal);
+        return out;
+    }
+    catch (err) {
+        throw settle(err, prepared);
+    }
+    finally {
+        prepared.created.dispose();
+    }
+}
+/**
+ * Add the `source` member to a report.
+ * @param report - the report
+ * @param prepared - the call
+ * @param next - where to resume, as the caller sees it
+ * @returns the report with its source member
+ */
+export function withSource(report, prepared, next) {
+    const own = prepared.report.finish();
+    // issues the context recorded (retries, limits) that a source's own report does not hold
+    const missing = own.issues.filter((i) => !report.issues.includes(i));
+    return Object.freeze({
+        ...report,
+        issues: missing.length === 0 ? report.issues : Object.freeze([...report.issues, ...missing]),
+        warningCount: report.warningCount + missing.filter((i) => i.severity === "warning").length,
+        errorCount: report.errorCount + missing.filter((i) => i.severity === "error").length,
+        source: prepared.created.info(next),
+    });
+}
+/**
+ * What a failed call throws: the signal's reason when it aborted, an ImportError with the source
+ * info attached, or the error itself.
+ * @param err - what was thrown
+ * @param prepared - the call
+ * @returns the error to throw
+ */
+function settle(err, prepared) {
+    const { signal } = prepared.created.ctx;
+    if (signal.aborted) {
+        return signal.reason;
+    }
+    if (err instanceof LimitStop) {
+        // a cap reached where nothing can be kept (a file cut short): the load fails with the cap
+        return settle(limitError(err, prepared), prepared);
+    }
+    if (err instanceof ImportError && err.report.source === undefined) {
+        const withInfo = new ImportError(err.message, { ...err.report, source: prepared.created.info(null) }, err.details);
+        withInfo.stack = err.stack;
+        return withInfo;
+    }
+    return err;
+}
+/**
+ * The E_SOURCE_LIMIT error of a cap that stopped a load which can keep nothing.
+ * @param stop - the cap reached
+ * @param prepared - the call
+ * @returns the error
+ */
+function limitError(stop, prepared) {
+    try {
+        return failOn(prepared.report, "E_SOURCE_LIMIT", stop.message, { limit: stop.limit, value: stop.value });
+    }
+    catch (err) {
+        return err;
+    }
+}
+/**
+ * The report of an import or expand source, or, when a cap stopped it under "stop" (a LimitStop
+ * raised by ctx.fetch that the source did not catch), the report of what it wrote so far.
+ * @param run - the source's call
+ * @param prepared - the call
+ * @returns the report
+ */
+export async function stopAtLimit(run, prepared) {
+    try {
+        return await run;
+    }
+    catch (err) {
+        if (err instanceof LimitStop) {
+            return prepared.report.finish();
+        }
+        throw err;
+    }
+}
+//# sourceMappingURL=run.js.map