@graphty/graph-io 0.3.47 → 0.3.48

package/src/sources/rest/params.ts

@@ -0,0 +1,243 @@
+/**
+ * The REST mapping format and the parameters of the `rest` source and its `openalex` preset (design
+ * sections 13.2 and 13.3). Experimental: the mapping carries `version: 1`. The REST work may add
+ * optional fields only.
+ */
+
+import { type SourceParamDescriptor } from "../types.js";
+
+/**
+ * How a paged request moves to its next page. Experimental.
+ * @category Network sources
+ */
+export type RestPaging =
+    | {
+          /** The paging style. */
+          readonly style: "cursor";
+          /** The query parameter that carries the position. */
+          readonly param: string;
+          /** The position of the first page. */
+          readonly first: string;
+          /** The selector of the next position in the body. */
+          readonly next: string;
+      }
+    | {
+          /** The paging style. */
+          readonly style: "offset";
+          /** The query parameter that carries the position. */
+          readonly param: string;
+          /** The page size and its query parameter. */
+          readonly size: {
+              /** The query parameter that carries the page size. */
+              readonly param: string;
+              /** The page size. */
+              readonly value: number;
+          };
+          /** Whether the offset counts items or pages. */
+          readonly unit?: "items" | "pages" | undefined;
+          /** The first offset. */
+          readonly base?: 0 | 1 | undefined;
+      }
+    | {
+          /** The paging style. */
+          readonly style: "link-header";
+          /** The link relation of the next page. */
+          readonly rel?: string | undefined;
+      }
+    | {
+          /** The paging style. */
+          readonly style: "next-url";
+          /** The selector of the next position in the body. */
+          readonly next: string;
+      };
+
+/**
+ * One named request of a mapping. Experimental.
+ * @category Network sources
+ */
+export interface RestRequest {
+    /** GET (default) or POST. */
+    readonly method?: "GET" | "POST" | undefined;
+    /** A path template: "/works/{id}". */
+    readonly path: string;
+    /** Query templates: `{ filter: "cites:{work}" }`. */
+    readonly query?: Readonly<Record<string, string>> | undefined;
+    /** A POST body; templates in string leaves. */
+    readonly body?: unknown;
+    /** The selector of the record array, "." for one record. */
+    readonly records: string;
+    /** Which entity the records are. */
+    readonly entity: string;
+    /** How it pages. */
+    readonly paging?: RestPaging | undefined;
+    /** `{ids}` holds up to `max` ids joined by `join`. */
+    readonly batch?:
+        | {
+              /** At most this many ids per request. */
+              readonly max: number;
+              /** What joins the ids. */
+              readonly join: string;
+          }
+        | undefined;
+}
+
+/**
+ * One rule that makes edges from a record. Experimental.
+ * @category Network sources
+ */
+export type RestEdgeRule =
+    | {
+          /** A selector the rule repeats over. */
+          readonly each?: string | undefined;
+          /** The selector of the edge's source; the record's id by default. */
+          readonly from?: string | undefined;
+          /** The selector of the edge's target ids. */
+          readonly to: string;
+          /** The edge type. */
+          readonly type: string;
+          /** The entity the target or record is. */
+          readonly entity?: string | undefined;
+          /** Attribute name to selector. */
+          readonly fields?: Readonly<Record<string, string>> | undefined;
+      }
+    | {
+          /** A selector the rule repeats over. */
+          readonly each?: string | undefined;
+          /** A selector of ids joined pairwise, undirected. */
+          readonly pairs: string;
+          /** The edge type. */
+          readonly type: string;
+          /** The entity the ids are, for its `stripPrefix` and for stubs; the record's own by default. */
+          readonly entity?: string | undefined;
+      };
+
+/**
+ * How records of one kind become nodes and edges. Experimental.
+ * @category Network sources
+ */
+export interface RestEntity {
+    /** The id selector within the record; `#` is the record's position in its array. */
+    readonly id: string;
+    /**
+     * false: the records are containers, not nodes; only their edge rules and embedded records
+     * count (a body holding a node array and an edge array).
+     */
+    readonly node?: false | undefined;
+    /** The node label; the entity's name by default. */
+    readonly label?: string | undefined;
+    /** A prefix stripped from ids ("https://openalex.org/"). */
+    readonly stripPrefix?: string | undefined;
+    /** Attribute name to selector; without it every top-level scalar is kept. */
+    readonly fields?: Readonly<Record<string, string>> | undefined;
+    /** The edge rules. */
+    readonly edges?: readonly RestEdgeRule[] | undefined;
+    /** Nested records that are nodes of their own entity. */
+    readonly embedded?:
+        | readonly {
+              /** The selector of the nested records. */
+              readonly at: string;
+              /** The entity the target or record is. */
+              readonly entity: string;
+          }[]
+        | undefined;
+}
+
+/**
+ * A declarative mapping from a REST API to a graph, as JSON data. Experimental.
+ * @category Network sources
+ */
+export interface RestMapping {
+    /** The format version. */
+    readonly version: 1;
+    /** Every request must stay on this origin. */
+    readonly endpoint: string;
+    /** The named requests. */
+    readonly requests: Readonly<Record<string, RestRequest>>;
+    /** How records become nodes and edges. */
+    readonly entities: Readonly<Record<string, RestEntity>>;
+    /** The request a load runs. */
+    readonly load: string;
+    /** The request an expansion runs per batch of seeds. */
+    readonly expand?:
+        | string
+        | {
+              /** The request for incoming edges. */
+              readonly in?: string | undefined;
+              /** The request for outgoing edges. */
+              readonly out?: string | undefined;
+          }
+        | undefined;
+    /** The request that fetches stub nodes' full records (an expansion of depth 0). */
+    readonly hydrate?: string | undefined;
+    /** A request run once before the load, for the seeds' own records. */
+    readonly seed?: string | undefined;
+    /** Whether the graph is directed; the builder's default when absent. */
+    readonly directed?: boolean | undefined;
+    /**
+     * The selector of a record's modification time (ISO-8601). The newest one loaded becomes
+     * `report.source.validators.modified`, which `validateSource` compares against.
+     */
+    readonly modified?: string | undefined;
+    /** What credential a form should ask for. */
+    readonly auth?:
+        | {
+              /** The credential kind. */
+              readonly kind: "bearer";
+          }
+        | {
+              /** The credential kind. */
+              readonly kind: "api-key";
+              /** The request header the key goes in. */
+              readonly header?: string | undefined;
+              /** The query parameter the key goes in. */
+              readonly query?: string | undefined;
+          }
+        | undefined;
+    /** Paces requests. */
+    readonly rateLimit?:
+        | {
+              /** At most this many requests per period. */
+              readonly requests: number;
+              /** The period, in milliseconds. */
+              readonly perMs: number;
+              /** At most this many requests at once. */
+              readonly concurrent?: number;
+          }
+        | undefined;
+    /** Response headers telling what budget is left. */
+    readonly budgetHeaders?:
+        | {
+              /** What the service says is left. */
+              readonly remaining: string;
+              /** The header naming when the budget resets. */
+              readonly reset?: string | undefined;
+          }
+        | undefined;
+    /** What a form asks for. */
+    readonly params?: readonly SourceParamDescriptor[] | undefined;
+}
+
+/**
+ * The params of the generic `rest` source: the mapping and the values its templates use.
+ * Experimental.
+ * @category Network sources
+ */
+export type RestParams = {
+    /** The mapping. */
+    readonly mapping: RestMapping;
+} & Readonly<Record<string, unknown>>;
+
+/**
+ * The params of the `openalex` preset. Experimental.
+ * @category Network sources
+ */
+export interface OpenAlexParams {
+    /** The seed work ("W2112090702"). */
+    readonly work?: string | undefined;
+    /** The seed author. */
+    readonly author?: string | undefined;
+    /** "citation" (default) or "coauthorship". */
+    readonly graph?: "citation" | "coauthorship" | undefined;
+    /** What else becomes nodes, as a list or a comma-separated string. @defaultValue [] */
+    readonly include?: readonly ("authors" | "institutions" | "topics")[] | string | undefined;
+}

package/src/sources/rest/records.ts

@@ -0,0 +1,289 @@
+/**
+ * Turning a REST API's records into nodes and edges (design section 13.2): each record of an
+ * entity is a node, each edge rule makes edges, a target with no record yet becomes a stub, and
+ * embedded records are nodes of their own entity. A stub never overwrites a full record.
+ */
+
+import { SOURCE_ENTITY_ATTR, SOURCE_STUB_ATTR } from "../ids.js";
+import { type EdgeRecord, type NodeRecord } from "../types.js";
+import { type RestEdgeRule, type RestEntity, type RestMapping } from "./params.js";
+import { select, selectOne } from "./selectors.js";
+
+/** What one load or expansion knows about the ids it wrote, across its pages. */
+export interface RecordState {
+    /** Ids written with a full record. */
+    readonly full: Set<string>;
+    /** Ids written as stubs. */
+    readonly stubs: Set<string>;
+    /** Ids the caller already holds (an expansion's seeds and `exclude`): never stubbed. */
+    readonly known: ReadonlySet<string>;
+    /** The newest modification time seen, for the validators. */
+    modified: string | null;
+}
+
+/** The nodes and edges of one page. */
+export interface PageRecords {
+    readonly nodes: NodeRecord[];
+    readonly edges: EdgeRecord[];
+}
+
+/**
+ * A fresh state.
+ * @param known - ids the caller holds already
+ * @returns the state
+ */
+export function newState(known: Iterable<string> = []): RecordState {
+    return { full: new Set(), stubs: new Set(), known: new Set(known), modified: null };
+}
+
+/**
+ * An id as text, with the entity's prefix stripped.
+ * @param value - what a selector reached
+ * @param entity - the entity whose prefix applies
+ * @returns the id, or null when the value is not one
+ */
+function idText(value: unknown, entity: RestEntity | undefined): string | null {
+    if (typeof value !== "string" && typeof value !== "number") {
+        return null;
+    }
+    const text = String(value);
+    const prefix = entity?.stripPrefix;
+    return prefix !== undefined && prefix !== "" && text.startsWith(prefix) ? text.slice(prefix.length) : text;
+}
+
+/**
+ * The value of a field selector: every value for a selector with `[]`, else the first.
+ * @param record - the record
+ * @param selector - the selector
+ * @param index - the record's position, for `#`
+ * @returns the value
+ */
+function fieldValue(record: unknown, selector: string, index: number): unknown {
+    if (selector === "#") {
+        return index;
+    }
+    return selector.includes("[]") ? select(record, selector) : selectOne(record, selector);
+}
+
+/** Converts the records of one page. */
+class PageWriter {
+    readonly out: PageRecords = { nodes: [], edges: [] };
+    /** Edge rules to run once every node of the page is known, so a target on this page is no stub. */
+    private readonly pending: { record: unknown; entity: string; id: string | null }[] = [];
+
+    /**
+     * Create one.
+     * @param mapping - the mapping
+     * @param state - the load's state
+     */
+    constructor(
+        private readonly mapping: RestMapping,
+        private readonly state: RecordState,
+    ) {}
+
+    /**
+     * Add the records of an entity, and their embedded records.
+     * @param records - the records
+     * @param entityName - their entity
+     */
+    addRecords(records: readonly unknown[], entityName: string): void {
+        const entity = this.mapping.entities[entityName];
+        records.forEach((record, index) => {
+            const id = entity.node === false ? null : idText(fieldValue(record, entity.id, index), entity);
+            if (entity.node !== false) {
+                if (id === null) {
+                    return;
+                }
+                this.addNode(record, index, id, entityName, entity);
+            }
+            const modified = this.mapping.modified === undefined ? undefined : selectOne(record, this.mapping.modified);
+            if (typeof modified === "string" && (this.state.modified === null || modified > this.state.modified)) {
+                this.state.modified = modified;
+            }
+            this.pending.push({ record, entity: entityName, id });
+            for (const em of entity.embedded ?? []) {
+                this.addRecords(select(record, em.at), em.entity);
+            }
+        });
+    }
+
+    /**
+     * Write a full record.
+     * @param record - the record
+     * @param index - its position
+     * @param id - its id
+     * @param entityName - its entity
+     * @param entity - the entity
+     */
+    private addNode(record: unknown, index: number, id: string, entityName: string, entity: RestEntity): void {
+        const attrs: Record<string, unknown> = { labels: [entity.label ?? entityName] };
+        if (entity.fields === undefined) {
+            // without fields every top-level scalar is kept
+            for (const [k, v] of Object.entries(record as Record<string, unknown>)) {
+                if (
+                    k !== "id" &&
+                    k !== "labels" &&
+                    (typeof v === "string" || typeof v === "number" || typeof v === "boolean")
+                ) {
+                    attrs[k] = v;
+                }
+            }
+        } else {
+            for (const [name, selector] of Object.entries(entity.fields)) {
+                const v = fieldValue(record, selector, index);
+                if (v !== undefined) {
+                    attrs[name] = v;
+                }
+            }
+        }
+        if (this.state.stubs.has(id)) {
+            // the full record replaces a stub written earlier
+            attrs[SOURCE_STUB_ATTR] = false;
+        }
+        this.state.full.add(id);
+        this.out.nodes.push({ id, ...attrs });
+    }
+
+    /**
+     * Make sure an edge end exists: a stub when no record of it was written and the caller does not
+     * hold it.
+     * @param id - the id
+     * @param entity - its entity, when known
+     */
+    private end(id: string, entity: string | undefined): void {
+        const { full, stubs, known } = this.state;
+        if (full.has(id) || stubs.has(id) || known.has(id)) {
+            return;
+        }
+        stubs.add(id);
+        this.out.nodes.push({
+            id,
+            [SOURCE_STUB_ATTR]: true,
+            ...(entity === undefined ? {} : { [SOURCE_ENTITY_ATTR]: entity }),
+        });
+    }
+
+    /**
+     * Run one edge rule over a record.
+     * @param rule - the rule
+     * @param record - the record
+     * @param entityName - the record's entity
+     * @param recordId - the record's id (null for a container)
+     */
+    private runRule(rule: RestEdgeRule, record: unknown, entityName: string, recordId: string | null): void {
+        const scopes = rule.each === undefined ? [record] : select(record, rule.each);
+        if ("pairs" in rule) {
+            this.runPairsRule(rule, scopes, entityName);
+        } else {
+            this.runLinkRule(rule, scopes, entityName, recordId);
+        }
+    }
+
+    /**
+     * Run a pairs rule: an edge between every two distinct ids in each scope.
+     * @param rule - the rule
+     * @param scopes - the scopes it runs over
+     * @param entityName - the record's entity
+     */
+    private runPairsRule(
+        rule: Extract<RestEdgeRule, { pairs: string }>,
+        scopes: readonly unknown[],
+        entityName: string,
+    ): void {
+        const entityOf = rule.entity ?? entityName;
+        const entity = this.mapping.entities[entityOf];
+        for (const scope of scopes) {
+            const ids = [...new Set(select(scope, rule.pairs).map((v) => idText(v, entity)))].filter(
+                (v): v is string => v !== null,
+            );
+            for (let i = 0; i < ids.length; i++) {
+                for (let j = i + 1; j < ids.length; j++) {
+                    this.end(ids[i], entityOf);
+                    this.end(ids[j], entityOf);
+                    this.out.edges.push({ source: ids[i], target: ids[j], type: rule.type });
+                }
+            }
+        }
+    }
+
+    /**
+     * Run a from/to rule: an edge from the record (or a selected id) to each selected id.
+     * @param rule - the rule
+     * @param scopes - the scopes it runs over
+     * @param entityName - the record's entity
+     * @param recordId - the record's id (null for a container)
+     */
+    private runLinkRule(
+        rule: Exclude<RestEdgeRule, { pairs: string }>,
+        scopes: readonly unknown[],
+        entityName: string,
+        recordId: string | null,
+    ): void {
+        const own = this.mapping.entities[entityName];
+        const targetEntity = rule.entity === undefined ? own : this.mapping.entities[rule.entity];
+        for (const scope of scopes) {
+            const from = rule.from === undefined ? recordId : idText(selectOne(scope, rule.from), own);
+            if (from === null) {
+                continue;
+            }
+            const tos = select(scope, rule.to)
+                .map((t) => idText(t, targetEntity))
+                .filter((v): v is string => v !== null);
+            for (const to of tos) {
+                this.end(from, rule.from === undefined ? entityName : undefined);
+                this.end(to, rule.entity);
+                this.out.edges.push({ ...edgeFields(rule.fields, scope), source: from, target: to, type: rule.type });
+            }
+        }
+    }
+
+    /**
+     * Run the edge rules of every record added.
+     * @returns the page
+     */
+    finish(): PageRecords {
+        for (const { record, entity, id } of this.pending) {
+            for (const rule of this.mapping.entities[entity].edges ?? []) {
+                this.runRule(rule, record, entity, id);
+            }
+        }
+        this.pending.length = 0;
+        return this.out;
+    }
+}
+
+/**
+ * An edge's fields, read from its scope.
+ * @param fields - the rule's field selectors
+ * @param scope - the scope the edge came from
+ * @returns the fields that have a value
+ */
+function edgeFields(fields: Readonly<Record<string, string>> | undefined, scope: unknown): Record<string, unknown> {
+    const out: Record<string, unknown> = {};
+    for (const [name, selector] of Object.entries(fields ?? {})) {
+        const v = fieldValue(scope, selector, 0);
+        if (v !== undefined) {
+            out[name] = v;
+        }
+    }
+    return out;
+}
+
+/**
+ * The nodes and edges of one page of records.
+ * @param mapping - the mapping
+ * @param groups - the page's records, by entity (a seed's records first)
+ * @param state - the load's state, updated
+ * @returns the nodes and edges
+ */
+export function toRecords(
+    mapping: RestMapping,
+    groups: readonly { readonly records: readonly unknown[]; readonly entity: string }[],
+    state: RecordState,
+): PageRecords {
+    const writer = new PageWriter(mapping, state);
+    for (const g of groups) {
+        writer.addRecords(g.records, g.entity);
+    }
+    return writer.finish();
+}

package/src/sources/rest/samples/deps-dev.ts

@@ -0,0 +1,49 @@
+/**
+ * A sample mapping: the resolved dependency graph of one package version from deps.dev
+ * (https://deps.dev). The body already holds a node array and an edge array whose ends are node
+ * positions, so the body is a container and `#` gives each package its position as its id.
+ * Catalog data for the `rest` source's samples.
+ */
+
+import { type RestMapping } from "../params.js";
+
+/** The deps.dev dependencies mapping: `system` (npm, pypi, ...), `package` and `version`. */
+export const DEPS_DEV_MAPPING: RestMapping = Object.freeze<RestMapping>({
+    version: 1,
+    endpoint: "https://api.deps.dev",
+    load: "dependencies",
+    directed: true,
+    requests: {
+        dependencies: {
+            path: "/v3/systems/{system}/packages/{package}/versions/{version}:dependencies",
+            records: ".",
+            entity: "graph",
+        },
+    },
+    entities: {
+        graph: {
+            id: "nodes",
+            node: false,
+            edges: [
+                {
+                    each: "edges[]",
+                    from: "fromNode",
+                    to: "toNode",
+                    type: "depends_on",
+                    entity: "package",
+                    fields: { requirement: "requirement" },
+                },
+            ],
+            embedded: [{ at: "nodes[]", entity: "package" }],
+        },
+        package: {
+            id: "#",
+            fields: {
+                name: "versionKey.name",
+                version: "versionKey.version",
+                system: "versionKey.system",
+                relation: "relation",
+            },
+        },
+    },
+});

package/src/sources/rest/samples/string.ts

@@ -0,0 +1,44 @@
+/**
+ * A sample mapping: STRING's functional partners of a protein (https://string-db.org). Each
+ * interaction record names both proteins, so both are embedded nodes and the record itself is only
+ * the edge. Catalog data for the `rest` source's samples.
+ */
+
+import { type RestMapping } from "../params.js";
+
+/** The STRING network mapping: `identifiers`, `species` (NCBI taxon) and `add_nodes`. */
+export const STRING_MAPPING: RestMapping = Object.freeze<RestMapping>({
+    version: 1,
+    endpoint: "https://string-db.org",
+    load: "network",
+    directed: false,
+    requests: {
+        network: {
+            path: "/api/json/network",
+            query: { identifiers: "{identifiers}", species: "{species}", add_nodes: "{add_nodes}" },
+            records: ".",
+            entity: "interaction",
+        },
+    },
+    entities: {
+        interaction: {
+            id: "stringId_A",
+            node: false,
+            edges: [
+                {
+                    from: "stringId_A",
+                    to: "stringId_B",
+                    type: "interacts_with",
+                    entity: "protein-b",
+                    fields: { score: "score" },
+                },
+            ],
+            embedded: [
+                { at: ".", entity: "protein-a" },
+                { at: ".", entity: "protein-b" },
+            ],
+        },
+        "protein-a": { id: "stringId_A", label: "protein", fields: { name: "preferredName_A" } },
+        "protein-b": { id: "stringId_B", label: "protein", fields: { name: "preferredName_B" } },
+    },
+});