@graphty/graph-io 0.3.47 → 0.3.49

package/src/sources/rest/records.ts

@@ -0,0 +1,289 @@
+/**
+ * Turning a REST API's records into nodes and edges (design section 13.2): each record of an
+ * entity is a node, each edge rule makes edges, a target with no record yet becomes a stub, and
+ * embedded records are nodes of their own entity. A stub never overwrites a full record.
+ */
+
+import { SOURCE_ENTITY_ATTR, SOURCE_STUB_ATTR } from "../ids.js";
+import { type EdgeRecord, type NodeRecord } from "../types.js";
+import { type RestEdgeRule, type RestEntity, type RestMapping } from "./params.js";
+import { select, selectOne } from "./selectors.js";
+
+/** What one load or expansion knows about the ids it wrote, across its pages. */
+export interface RecordState {
+    /** Ids written with a full record. */
+    readonly full: Set<string>;
+    /** Ids written as stubs. */
+    readonly stubs: Set<string>;
+    /** Ids the caller already holds (an expansion's seeds and `exclude`): never stubbed. */
+    readonly known: ReadonlySet<string>;
+    /** The newest modification time seen, for the validators. */
+    modified: string | null;
+}
+
+/** The nodes and edges of one page. */
+export interface PageRecords {
+    readonly nodes: NodeRecord[];
+    readonly edges: EdgeRecord[];
+}
+
+/**
+ * A fresh state.
+ * @param known - ids the caller holds already
+ * @returns the state
+ */
+export function newState(known: Iterable<string> = []): RecordState {
+    return { full: new Set(), stubs: new Set(), known: new Set(known), modified: null };
+}
+
+/**
+ * An id as text, with the entity's prefix stripped.
+ * @param value - what a selector reached
+ * @param entity - the entity whose prefix applies
+ * @returns the id, or null when the value is not one
+ */
+function idText(value: unknown, entity: RestEntity | undefined): string | null {
+    if (typeof value !== "string" && typeof value !== "number") {
+        return null;
+    }
+    const text = String(value);
+    const prefix = entity?.stripPrefix;
+    return prefix !== undefined && prefix !== "" && text.startsWith(prefix) ? text.slice(prefix.length) : text;
+}
+
+/**
+ * The value of a field selector: every value for a selector with `[]`, else the first.
+ * @param record - the record
+ * @param selector - the selector
+ * @param index - the record's position, for `#`
+ * @returns the value
+ */
+function fieldValue(record: unknown, selector: string, index: number): unknown {
+    if (selector === "#") {
+        return index;
+    }
+    return selector.includes("[]") ? select(record, selector) : selectOne(record, selector);
+}
+
+/** Converts the records of one page. */
+class PageWriter {
+    readonly out: PageRecords = { nodes: [], edges: [] };
+    /** Edge rules to run once every node of the page is known, so a target on this page is no stub. */
+    private readonly pending: { record: unknown; entity: string; id: string | null }[] = [];
+
+    /**
+     * Create one.
+     * @param mapping - the mapping
+     * @param state - the load's state
+     */
+    constructor(
+        private readonly mapping: RestMapping,
+        private readonly state: RecordState,
+    ) {}
+
+    /**
+     * Add the records of an entity, and their embedded records.
+     * @param records - the records
+     * @param entityName - their entity
+     */
+    addRecords(records: readonly unknown[], entityName: string): void {
+        const entity = this.mapping.entities[entityName];
+        records.forEach((record, index) => {
+            const id = entity.node === false ? null : idText(fieldValue(record, entity.id, index), entity);
+            if (entity.node !== false) {
+                if (id === null) {
+                    return;
+                }
+                this.addNode(record, index, id, entityName, entity);
+            }
+            const modified = this.mapping.modified === undefined ? undefined : selectOne(record, this.mapping.modified);
+            if (typeof modified === "string" && (this.state.modified === null || modified > this.state.modified)) {
+                this.state.modified = modified;
+            }
+            this.pending.push({ record, entity: entityName, id });
+            for (const em of entity.embedded ?? []) {
+                this.addRecords(select(record, em.at), em.entity);
+            }
+        });
+    }
+
+    /**
+     * Write a full record.
+     * @param record - the record
+     * @param index - its position
+     * @param id - its id
+     * @param entityName - its entity
+     * @param entity - the entity
+     */
+    private addNode(record: unknown, index: number, id: string, entityName: string, entity: RestEntity): void {
+        const attrs: Record<string, unknown> = { labels: [entity.label ?? entityName] };
+        if (entity.fields === undefined) {
+            // without fields every top-level scalar is kept
+            for (const [k, v] of Object.entries(record as Record<string, unknown>)) {
+                if (
+                    k !== "id" &&
+                    k !== "labels" &&
+                    (typeof v === "string" || typeof v === "number" || typeof v === "boolean")
+                ) {
+                    attrs[k] = v;
+                }
+            }
+        } else {
+            for (const [name, selector] of Object.entries(entity.fields)) {
+                const v = fieldValue(record, selector, index);
+                if (v !== undefined) {
+                    attrs[name] = v;
+                }
+            }
+        }
+        if (this.state.stubs.has(id)) {
+            // the full record replaces a stub written earlier
+            attrs[SOURCE_STUB_ATTR] = false;
+        }
+        this.state.full.add(id);
+        this.out.nodes.push({ id, ...attrs });
+    }
+
+    /**
+     * Make sure an edge end exists: a stub when no record of it was written and the caller does not
+     * hold it.
+     * @param id - the id
+     * @param entity - its entity, when known
+     */
+    private end(id: string, entity: string | undefined): void {
+        const { full, stubs, known } = this.state;
+        if (full.has(id) || stubs.has(id) || known.has(id)) {
+            return;
+        }
+        stubs.add(id);
+        this.out.nodes.push({
+            id,
+            [SOURCE_STUB_ATTR]: true,
+            ...(entity === undefined ? {} : { [SOURCE_ENTITY_ATTR]: entity }),
+        });
+    }
+
+    /**
+     * Run one edge rule over a record.
+     * @param rule - the rule
+     * @param record - the record
+     * @param entityName - the record's entity
+     * @param recordId - the record's id (null for a container)
+     */
+    private runRule(rule: RestEdgeRule, record: unknown, entityName: string, recordId: string | null): void {
+        const scopes = rule.each === undefined ? [record] : select(record, rule.each);
+        if ("pairs" in rule) {
+            this.runPairsRule(rule, scopes, entityName);
+        } else {
+            this.runLinkRule(rule, scopes, entityName, recordId);
+        }
+    }
+
+    /**
+     * Run a pairs rule: an edge between every two distinct ids in each scope.
+     * @param rule - the rule
+     * @param scopes - the scopes it runs over
+     * @param entityName - the record's entity
+     */
+    private runPairsRule(
+        rule: Extract<RestEdgeRule, { pairs: string }>,
+        scopes: readonly unknown[],
+        entityName: string,
+    ): void {
+        const entityOf = rule.entity ?? entityName;
+        const entity = this.mapping.entities[entityOf];
+        for (const scope of scopes) {
+            const ids = [...new Set(select(scope, rule.pairs).map((v) => idText(v, entity)))].filter(
+                (v): v is string => v !== null,
+            );
+            for (let i = 0; i < ids.length; i++) {
+                for (let j = i + 1; j < ids.length; j++) {
+                    this.end(ids[i], entityOf);
+                    this.end(ids[j], entityOf);
+                    this.out.edges.push({ source: ids[i], target: ids[j], type: rule.type });
+                }
+            }
+        }
+    }
+
+    /**
+     * Run a from/to rule: an edge from the record (or a selected id) to each selected id.
+     * @param rule - the rule
+     * @param scopes - the scopes it runs over
+     * @param entityName - the record's entity
+     * @param recordId - the record's id (null for a container)
+     */
+    private runLinkRule(
+        rule: Exclude<RestEdgeRule, { pairs: string }>,
+        scopes: readonly unknown[],
+        entityName: string,
+        recordId: string | null,
+    ): void {
+        const own = this.mapping.entities[entityName];
+        const targetEntity = rule.entity === undefined ? own : this.mapping.entities[rule.entity];
+        for (const scope of scopes) {
+            const from = rule.from === undefined ? recordId : idText(selectOne(scope, rule.from), own);
+            if (from === null) {
+                continue;
+            }
+            const tos = select(scope, rule.to)
+                .map((t) => idText(t, targetEntity))
+                .filter((v): v is string => v !== null);
+            for (const to of tos) {
+                this.end(from, rule.from === undefined ? entityName : undefined);
+                this.end(to, rule.entity);
+                this.out.edges.push({ ...edgeFields(rule.fields, scope), source: from, target: to, type: rule.type });
+            }
+        }
+    }
+
+    /**
+     * Run the edge rules of every record added.
+     * @returns the page
+     */
+    finish(): PageRecords {
+        for (const { record, entity, id } of this.pending) {
+            for (const rule of this.mapping.entities[entity].edges ?? []) {
+                this.runRule(rule, record, entity, id);
+            }
+        }
+        this.pending.length = 0;
+        return this.out;
+    }
+}
+
+/**
+ * An edge's fields, read from its scope.
+ * @param fields - the rule's field selectors
+ * @param scope - the scope the edge came from
+ * @returns the fields that have a value
+ */
+function edgeFields(fields: Readonly<Record<string, string>> | undefined, scope: unknown): Record<string, unknown> {
+    const out: Record<string, unknown> = {};
+    for (const [name, selector] of Object.entries(fields ?? {})) {
+        const v = fieldValue(scope, selector, 0);
+        if (v !== undefined) {
+            out[name] = v;
+        }
+    }
+    return out;
+}
+
+/**
+ * The nodes and edges of one page of records.
+ * @param mapping - the mapping
+ * @param groups - the page's records, by entity (a seed's records first)
+ * @param state - the load's state, updated
+ * @returns the nodes and edges
+ */
+export function toRecords(
+    mapping: RestMapping,
+    groups: readonly { readonly records: readonly unknown[]; readonly entity: string }[],
+    state: RecordState,
+): PageRecords {
+    const writer = new PageWriter(mapping, state);
+    for (const g of groups) {
+        writer.addRecords(g.records, g.entity);
+    }
+    return writer.finish();
+}

package/src/sources/rest/samples/deps-dev.ts

@@ -0,0 +1,49 @@
+/**
+ * A sample mapping: the resolved dependency graph of one package version from deps.dev
+ * (https://deps.dev). The body already holds a node array and an edge array whose ends are node
+ * positions, so the body is a container and `#` gives each package its position as its id.
+ * Catalog data for the `rest` source's samples.
+ */
+
+import { type RestMapping } from "../params.js";
+
+/** The deps.dev dependencies mapping: `system` (npm, pypi, ...), `package` and `version`. */
+export const DEPS_DEV_MAPPING: RestMapping = Object.freeze<RestMapping>({
+    version: 1,
+    endpoint: "https://api.deps.dev",
+    load: "dependencies",
+    directed: true,
+    requests: {
+        dependencies: {
+            path: "/v3/systems/{system}/packages/{package}/versions/{version}:dependencies",
+            records: ".",
+            entity: "graph",
+        },
+    },
+    entities: {
+        graph: {
+            id: "nodes",
+            node: false,
+            edges: [
+                {
+                    each: "edges[]",
+                    from: "fromNode",
+                    to: "toNode",
+                    type: "depends_on",
+                    entity: "package",
+                    fields: { requirement: "requirement" },
+                },
+            ],
+            embedded: [{ at: "nodes[]", entity: "package" }],
+        },
+        package: {
+            id: "#",
+            fields: {
+                name: "versionKey.name",
+                version: "versionKey.version",
+                system: "versionKey.system",
+                relation: "relation",
+            },
+        },
+    },
+});

package/src/sources/rest/samples/string.ts

@@ -0,0 +1,44 @@
+/**
+ * A sample mapping: STRING's functional partners of a protein (https://string-db.org). Each
+ * interaction record names both proteins, so both are embedded nodes and the record itself is only
+ * the edge. Catalog data for the `rest` source's samples.
+ */
+
+import { type RestMapping } from "../params.js";
+
+/** The STRING network mapping: `identifiers`, `species` (NCBI taxon) and `add_nodes`. */
+export const STRING_MAPPING: RestMapping = Object.freeze<RestMapping>({
+    version: 1,
+    endpoint: "https://string-db.org",
+    load: "network",
+    directed: false,
+    requests: {
+        network: {
+            path: "/api/json/network",
+            query: { identifiers: "{identifiers}", species: "{species}", add_nodes: "{add_nodes}" },
+            records: ".",
+            entity: "interaction",
+        },
+    },
+    entities: {
+        interaction: {
+            id: "stringId_A",
+            node: false,
+            edges: [
+                {
+                    from: "stringId_A",
+                    to: "stringId_B",
+                    type: "interacts_with",
+                    entity: "protein-b",
+                    fields: { score: "score" },
+                },
+            ],
+            embedded: [
+                { at: ".", entity: "protein-a" },
+                { at: ".", entity: "protein-b" },
+            ],
+        },
+        "protein-a": { id: "stringId_A", label: "protein", fields: { name: "preferredName_A" } },
+        "protein-b": { id: "stringId_B", label: "protein", fields: { name: "preferredName_B" } },
+    },
+});

package/src/sources/rest/selectors.ts

@@ -0,0 +1,91 @@
+/**
+ * Selectors of the REST mapping (design section 13.2): graph-io's dotted paths plus `[]` to flatten
+ * an array. `results`, `meta.next_cursor`, `authorships[].author.id`; `.` is the value itself and
+ * `#` is a record's position in the array it came from. Not JMESPath, so graph-io keeps zero
+ * dependencies.
+ */
+
+/** One step of a selector: a key, and whether its value's array is flattened. */
+interface Step {
+    readonly key: string;
+    readonly flatten: boolean;
+}
+
+/** A key: anything but a dot or a bracket. */
+const STEP = /^([^.[\]]+)(\[\])?$/;
+
+/** Parsed selectors, by text. */
+const PARSED = new Map<string, readonly Step[] | null>();
+
+/**
+ * Parse a selector.
+ * @param selector - the text
+ * @returns its steps ([] for "."), or null when it is not a selector
+ */
+export function parseSelector(selector: string): readonly Step[] | null {
+    const cached = PARSED.get(selector);
+    if (cached !== undefined) {
+        return cached;
+    }
+    let steps: Step[] | null = [];
+    if (selector !== ".") {
+        for (const part of selector.split(".")) {
+            const m = STEP.exec(part);
+            if (m === null) {
+                steps = null;
+                break;
+            }
+            steps.push({ key: m[1], flatten: m[2] !== undefined });
+        }
+    }
+    PARSED.set(selector, steps);
+    return steps;
+}
+
+/**
+ * Every value a selector reaches, in document order; missing keys reach nothing.
+ * @param value - where to start
+ * @param selector - the selector (a checked mapping holds only valid ones)
+ * @returns the values
+ */
+export function select(value: unknown, selector: string): unknown[] {
+    let out: unknown[] = value === undefined ? [] : [value];
+    for (const step of parseSelector(selector) ?? []) {
+        const next: unknown[] = [];
+        for (const v of out) {
+            if (typeof v !== "object" || v === null || !Object.hasOwn(v, step.key)) {
+                continue;
+            }
+            const got = (v as Record<string, unknown>)[step.key];
+            if (step.flatten && Array.isArray(got)) {
+                next.push(...(got as unknown[]));
+            } else if (got !== undefined && got !== null) {
+                next.push(got);
+            }
+        }
+        out = next;
+    }
+    return out.filter((v) => v !== null && v !== undefined);
+}
+
+/**
+ * The first value a selector reaches.
+ * @param value - where to start
+ * @param selector - the selector
+ * @returns the value, or undefined
+ */
+export function selectOne(value: unknown, selector: string): unknown {
+    return select(value, selector)[0];
+}
+
+/**
+ * The records a request's `records` selector names: an array it reaches is the records, any other
+ * value is one record.
+ * @param body - the response body
+ * @param selector - the selector
+ * @returns the records
+ */
+export function selectRecords(body: unknown, selector: string): unknown[] {
+    const out = select(body, selector);
+    return out.length === 1 && Array.isArray(out[0]) ? (out[0] as unknown[]) : out;
+}