@graphty/graph-io 0.3.47 → 0.3.49

package/src/sources/rest/mapping.ts

@@ -0,0 +1,401 @@
+/**
+ * Checking a REST mapping before it runs (design section 13.2). A mapping is JSON data, typed into
+ * a form or read from a project file, so every field is checked and every problem is a coded
+ * issue naming where it is.
+ */
+
+import { type RestEdgeRule, type RestEntity, type RestMapping, type RestRequest } from "./params.js";
+import { parseSelector } from "./selectors.js";
+import { templateNames } from "./templates.js";
+
+/**
+ * What is wrong with one field of a mapping. Open union.
+ * @category Network sources
+ */
+export type RestMappingIssueCode =
+    | "version" // version is not 1
+    | "endpoint" // not an http(s) URL, or one holding a user, query or fragment
+    | "missing" // a required field is absent or of the wrong type
+    | "unknown-request" // names a request the mapping lacks
+    | "unknown-entity" // names an entity the mapping lacks
+    | "selector" // not a selector
+    | "template" // malformed braces or slot names
+    | "paging" // an unknown paging style or a bad paging field
+    | "batch" // batch.max is not a positive integer, or join is not a string
+    | "number" // not a positive number
+    | (string & {}); // NOSONAR(S4335): the open-union idiom keeps the listed codes in editor completion
+
+/**
+ * One problem `checkRestMapping` found. Experimental.
+ * @category Network sources
+ */
+export interface RestMappingIssue {
+    /** What is wrong. */
+    readonly code: RestMappingIssueCode;
+    /** Where, as a dotted path into the mapping (`requests.citing.paging.next`). */
+    readonly at: string;
+}
+
+/** The checks' accumulator. */
+type Push = (code: RestMappingIssueCode, at: string) => void;
+
+/**
+ * Whether a value is a plain object.
+ * @param v - the value
+ * @returns true for an object that is not an array
+ */
+function isObject(v: unknown): v is Record<string, unknown> {
+    return typeof v === "object" && v !== null && !Array.isArray(v);
+}
+
+/**
+ * Check a selector field.
+ * @param value - the field
+ * @param at - where it is
+ * @param push - the accumulator
+ * @param required - whether it must be present
+ * @param index - whether `#` (the record's position) is allowed
+ */
+function checkSelector(value: unknown, at: string, push: Push, required: boolean, index = false): void {
+    if (value === undefined && !required) {
+        return;
+    }
+    if (typeof value !== "string") {
+        push("missing", at);
+    } else if (!(index && value === "#") && parseSelector(value) === null) {
+        push("selector", at);
+    }
+}
+
+/**
+ * Check a template field.
+ * @param value - the field
+ * @param at - where it is
+ * @param push - the accumulator
+ */
+function checkTemplate(value: unknown, at: string, push: Push): void {
+    if (typeof value !== "string") {
+        push("missing", at);
+    } else if (templateNames(value) === null) {
+        push("template", at);
+    }
+}
+
+/**
+ * Check a request's query templates.
+ * @param query - the query field
+ * @param at - where the request is
+ * @param push - the accumulator
+ */
+function checkQuery(query: RestRequest["query"], at: string, push: Push): void {
+    if (query === undefined) {
+        return;
+    }
+    if (!isObject(query)) {
+        push("missing", `${at}.query`);
+        return;
+    }
+    for (const [k, v] of Object.entries(query)) {
+        checkTemplate(v, `${at}.query.${k}`, push);
+    }
+}
+
+/**
+ * Whether a batch field is well formed: a positive integer max and a string join.
+ * @param batch - the batch field
+ * @returns true when it is
+ */
+function isBatch(batch: RestRequest["batch"]): boolean {
+    return isObject(batch) && Number.isInteger(batch.max) && batch.max >= 1 && typeof batch.join === "string";
+}
+
+/**
+ * Check one request.
+ * @param r - the request
+ * @param at - where it is
+ * @param entities - the mapping's entity names
+ * @param push - the accumulator
+ */
+function checkRequest(r: RestRequest, at: string, entities: ReadonlySet<string>, push: Push): void {
+    if (!isObject(r)) {
+        push("missing", at);
+        return;
+    }
+    if (r.method !== undefined && r.method !== "GET" && r.method !== "POST") {
+        push("missing", `${at}.method`);
+    }
+    checkTemplate(r.path, `${at}.path`, push);
+    if (typeof r.path === "string" && !r.path.startsWith("/")) {
+        push("template", `${at}.path`);
+    }
+    checkQuery(r.query, at, push);
+    checkSelector(r.records, `${at}.records`, push, true);
+    if (typeof r.entity !== "string") {
+        push("missing", `${at}.entity`);
+    } else if (!entities.has(r.entity)) {
+        push("unknown-entity", `${at}.entity`);
+    }
+    if (r.batch !== undefined && !isBatch(r.batch)) {
+        push("batch", `${at}.batch`);
+    }
+    if (r.paging !== undefined) {
+        checkPaging(r.paging, `${at}.paging`, push);
+    }
+}
+
+/**
+ * Check a paging field.
+ * @param p - the paging
+ * @param at - where it is
+ * @param push - the accumulator
+ */
+function checkPaging(p: RestRequest["paging"], at: string, push: Push): void {
+    const paging = p as Record<string, unknown> | undefined;
+    if (!isObject(paging)) {
+        push("paging", at);
+        return;
+    }
+    switch (paging.style) {
+        case "cursor":
+            if (typeof paging.param !== "string" || typeof paging.first !== "string") {
+                push("paging", at);
+            }
+            checkSelector(paging.next, `${at}.next`, push, true);
+            break;
+        case "offset": {
+            const size = paging.size as Record<string, unknown> | undefined;
+            const ok =
+                typeof paging.param === "string" &&
+                isObject(size) &&
+                typeof size.param === "string" &&
+                Number.isInteger(size.value) &&
+                (size.value as number) > 0 &&
+                (paging.unit === undefined || paging.unit === "items" || paging.unit === "pages") &&
+                (paging.base === undefined || paging.base === 0 || paging.base === 1);
+            if (!ok) {
+                push("paging", at);
+            }
+            break;
+        }
+        case "link-header":
+            if (paging.rel !== undefined && typeof paging.rel !== "string") {
+                push("paging", at);
+            }
+            break;
+        case "next-url":
+            checkSelector(paging.next, `${at}.next`, push, true);
+            break;
+        default:
+            push("paging", `${at}.style`);
+    }
+}
+
+/**
+ * Check one edge rule.
+ * @param rule - the rule
+ * @param at - where it is
+ * @param entities - the mapping's entity names
+ * @param push - the accumulator
+ */
+function checkEdgeRule(rule: RestEdgeRule, at: string, entities: ReadonlySet<string>, push: Push): void {
+    if (!isObject(rule)) {
+        push("missing", at);
+        return;
+    }
+    const r = rule as Record<string, unknown>;
+    if (typeof r.type !== "string") {
+        push("missing", `${at}.type`);
+    }
+    checkSelector(r.each, `${at}.each`, push, false);
+    if ("pairs" in r) {
+        checkSelector(r.pairs, `${at}.pairs`, push, true);
+    } else {
+        checkSelector(r.to, `${at}.to`, push, true);
+        checkSelector(r.from, `${at}.from`, push, false);
+        if (isObject(r.fields)) {
+            for (const [k, v] of Object.entries(r.fields)) {
+                checkSelector(v, `${at}.fields.${k}`, push, true);
+            }
+        } else if (r.fields !== undefined) {
+            push("missing", `${at}.fields`);
+        }
+    }
+    if (r.entity !== undefined && (typeof r.entity !== "string" || !entities.has(r.entity))) {
+        push("unknown-entity", `${at}.entity`);
+    }
+}
+
+/**
+ * Check one entity.
+ * @param e - the entity
+ * @param at - where it is
+ * @param entities - the mapping's entity names
+ * @param push - the accumulator
+ */
+function checkEntity(e: RestEntity, at: string, entities: ReadonlySet<string>, push: Push): void {
+    if (!isObject(e)) {
+        push("missing", at);
+        return;
+    }
+    checkSelector(e.id, `${at}.id`, push, true, true);
+    if (e.stripPrefix !== undefined && typeof e.stripPrefix !== "string") {
+        push("missing", `${at}.stripPrefix`);
+    }
+    if (e.label !== undefined && typeof e.label !== "string") {
+        push("missing", `${at}.label`);
+    }
+    if (e.node !== undefined && e.node) {
+        push("missing", `${at}.node`);
+    }
+    if (isObject(e.fields)) {
+        for (const [k, v] of Object.entries(e.fields)) {
+            checkSelector(v, `${at}.fields.${k}`, push, true, true);
+        }
+    } else if (e.fields !== undefined) {
+        push("missing", `${at}.fields`);
+    }
+    if (Array.isArray(e.edges)) {
+        (e.edges as readonly RestEdgeRule[]).forEach((rule, i) => {
+            checkEdgeRule(rule, `${at}.edges.${String(i)}`, entities, push);
+        });
+    } else if (e.edges !== undefined) {
+        push("missing", `${at}.edges`);
+    }
+    if (Array.isArray(e.embedded)) {
+        e.embedded.forEach((em, i) => {
+            const where = `${at}.embedded.${String(i)}`;
+            if (!isObject(em)) {
+                push("missing", where);
+                return;
+            }
+            checkSelector(em.at, `${where}.at`, push, true);
+            if (typeof em.entity !== "string" || !entities.has(em.entity)) {
+                push("unknown-entity", `${where}.entity`);
+            }
+        });
+    } else if (e.embedded !== undefined) {
+        push("missing", `${at}.embedded`);
+    }
+}
+
+/**
+ * Check the mapping's top-level names of requests.
+ * @param m - the mapping
+ * @param requests - its request names
+ * @param push - the accumulator
+ */
+function checkRequestNames(m: RestMapping, requests: ReadonlySet<string>, push: Push): void {
+    const names: [string, unknown][] = [
+        ["load", m.load],
+        ["hydrate", m.hydrate],
+        ["seed", m.seed],
+    ];
+    if (isObject(m.expand)) {
+        names.push(["expand.in", m.expand.in], ["expand.out", m.expand.out]);
+    } else {
+        names.push(["expand", m.expand]);
+    }
+    for (const [at, name] of names) {
+        if (name === undefined && at !== "load") {
+            continue;
+        }
+        if (typeof name !== "string") {
+            push("missing", at);
+        } else if (!requests.has(name)) {
+            push("unknown-request", at);
+        }
+    }
+}
+
+/**
+ * Check the endpoint.
+ * @param endpoint - the field
+ * @param push - the accumulator
+ */
+function checkEndpoint(endpoint: unknown, push: Push): void {
+    let url: URL | null = null;
+    try {
+        url = new URL(typeof endpoint === "string" ? endpoint : "");
+    } catch {
+        // not a URL
+    }
+    if (
+        url === null ||
+        (url.protocol !== "https:" && url.protocol !== "http:") ||
+        url.username !== "" ||
+        url.password !== "" ||
+        url.search !== "" ||
+        url.hash !== ""
+    ) {
+        push("endpoint", "endpoint");
+    }
+}
+
+/**
+ * Check a REST mapping without running it: the version, the endpoint, every request (templates,
+ * selectors, paging, batching) and entity (selectors, edge rules, embedded records), and that every
+ * name it uses exists. Experimental.
+ * @param mapping - the mapping, as data from anywhere
+ * @returns every problem found; empty when the mapping can run
+ * @category Network sources
+ */
+export function checkRestMapping(mapping: unknown): readonly RestMappingIssue[] {
+    const issues: RestMappingIssue[] = [];
+    const push: Push = (code, at) => {
+        issues.push(Object.freeze({ code, at }));
+    };
+    if (!isObject(mapping)) {
+        push("missing", "");
+        return issues;
+    }
+    const m = mapping as unknown as RestMapping;
+    if (m.version !== 1) {
+        push("version", "version");
+    }
+    checkEndpoint(m.endpoint, push);
+    const requests = isObject(m.requests) ? m.requests : {};
+    const entities = isObject(m.entities) ? m.entities : {};
+    if (!isObject(m.requests)) {
+        push("missing", "requests");
+    }
+    if (!isObject(m.entities)) {
+        push("missing", "entities");
+    }
+    const entityNames = new Set(Object.keys(entities));
+    for (const [name, r] of Object.entries(requests)) {
+        checkRequest(r, `requests.${name}`, entityNames, push);
+    }
+    for (const [name, e] of Object.entries(entities)) {
+        checkEntity(e, `entities.${name}`, entityNames, push);
+    }
+    checkRequestNames(m, new Set(Object.keys(requests)), push);
+    checkLimits(m, push);
+    checkSelector(m.modified, "modified", push, false);
+    if (m.directed !== undefined && typeof m.directed !== "boolean") {
+        push("missing", "directed");
+    }
+    return issues;
+}
+
+/**
+ * Check the rate limit and the budget headers.
+ * @param m - the mapping
+ * @param push - the accumulator
+ */
+function checkLimits(m: RestMapping, push: Push): void {
+    for (const [at, value] of [
+        ["rateLimit.requests", m.rateLimit?.requests],
+        ["rateLimit.perMs", m.rateLimit?.perMs],
+    ] as const) {
+        if (m.rateLimit !== undefined && !(typeof value === "number" && value > 0)) {
+            push("number", at);
+        }
+    }
+    if (
+        m.budgetHeaders !== undefined &&
+        (typeof m.budgetHeaders.remaining !== "string" ||
+            (m.budgetHeaders.reset !== undefined && typeof m.budgetHeaders.reset !== "string"))
+    ) {
+        push("missing", "budgetHeaders");
+    }
+}

package/src/sources/rest/openalex.ts

@@ -0,0 +1,288 @@
+/**
+ * The `openalex` preset of the REST mapping (design section 13.3): OpenAlex's citation graph
+ * around a work, or its co-authorship graph around an author, as a mapping the generic engine runs.
+ * The seed's own record comes first; citing works page by cursor; referenced works are stubs until
+ * hydrated or expanded. Freshness is the newest `updated_date`.
+ */
+
+import { defineSource } from "../define.js";
+import { failOn } from "../fail.js";
+import { OPENALEX_SOURCE_ID } from "../ids.js";
+import { type GraphSource, type SourceContext } from "../types.js";
+import { OPENALEX_DESCRIPTOR } from "./descriptor.js";
+import { type OpenAlexParams, type RestEntity, type RestMapping } from "./params.js";
+import { restMethods, type RestPlan, type RestPurpose } from "./source.js";
+
+/** OpenAlex ids are URLs under this prefix. */
+const PREFIX = "https://openalex.org/";
+
+/** The work fields every citation load selects. */
+const WORK_FIELDS = "id,display_name,publication_year,cited_by_count,updated_date,referenced_works";
+
+/**
+ * A list request over works, `{ids}` batched 50 at a time and joined by `|`.
+ * @param filter - the filter template
+ * @param perPage - the page size
+ * @param paged - whether it pages by cursor
+ * @returns the request
+ */
+function works(filter: string, perPage: string, paged: boolean): RestMapping["requests"][string] {
+    return {
+        path: "/works",
+        query: { filter, "per-page": perPage, select: "{select}" },
+        batch: { max: 50, join: "|" },
+        records: "results",
+        entity: "work",
+        ...(paged ? { paging: { style: "cursor", param: "cursor", first: "*", next: "meta.next_cursor" } } : {}),
+    };
+}
+
+/** An entity known by its OpenAlex id and display name. */
+const named: RestEntity = { id: "id", stripPrefix: PREFIX, fields: { name: "display_name" } };
+
+/**
+ * OpenAlex's citation graph as a mapping: the seed work (`{ids}`), the works citing it page by page,
+ * each work's references as `cites` edges, and, when `{select}` asks for `authorships` and
+ * `topics`, the authors, their institutions and the topics as nodes. Fill `{ids}` with work ids
+ * and `{select}` with the fields to read. The `openalex` source chooses both from its params and
+ * leaves out the rules for what is not included. Experimental.
+ * @category Network sources
+ */
+export const OPENALEX_MAPPING: RestMapping = Object.freeze<RestMapping>({
+    version: 1,
+    endpoint: "https://api.openalex.org",
+    load: "citing",
+    seed: "by-id",
+    expand: { in: "citing", out: "by-id" },
+    hydrate: "by-id",
+    directed: true,
+    modified: "updated_date",
+    auth: { kind: "api-key", query: "api_key" },
+    rateLimit: { requests: 10, perMs: 1000 },
+    budgetHeaders: { remaining: "x-ratelimit-remaining", reset: "x-ratelimit-reset" },
+    requests: {
+        citing: works("cites:{ids}", "200", true),
+        "by-id": works("openalex:{ids}", "50", false),
+        authored: works("authorships.author.id:{ids}", "200", true),
+        "authors-by-id": {
+            path: "/authors",
+            query: { filter: "openalex:{ids}", "per-page": "50", select: "id,display_name" },
+            batch: { max: 50, join: "|" },
+            records: "results",
+            entity: "author",
+        },
+    },
+    entities: {
+        work: {
+            id: "id",
+            stripPrefix: PREFIX,
+            fields: {
+                title: "display_name",
+                year: "publication_year",
+                citedBy: "cited_by_count",
+                updated: "updated_date",
+            },
+            edges: [
+                { to: "referenced_works[]", type: "cites", entity: "work" },
+                { to: "authorships[].author.id", type: "authored_by", entity: "author" },
+                {
+                    each: "authorships[]",
+                    from: "author.id",
+                    to: "institutions[].id",
+                    type: "affiliated_with",
+                    entity: "institution",
+                },
+                { to: "topics[].id", type: "has_topic", entity: "topic" },
+            ],
+            embedded: [
+                { at: "authorships[].author", entity: "author" },
+                { at: "authorships[].institutions[]", entity: "institution" },
+                { at: "topics[]", entity: "topic" },
+            ],
+        },
+        author: named,
+        institution: named,
+        topic: named,
+    },
+});
+
+/** What `include` may hold. */
+type Include = "authors" | "institutions" | "topics";
+
+/**
+ * The included kinds, from an array or a comma-separated string.
+ * @param include - the param
+ * @returns the set
+ */
+function includes(include: unknown): Set<Include> {
+    const list: unknown[] = typeof include === "string" ? include.split(",") : [];
+    if (Array.isArray(include)) {
+        list.push(...(include as unknown[]));
+    }
+    const out = new Set(list.map((v) => String(v).trim()).filter((v) => v !== "") as Include[]);
+    if (out.has("institutions")) {
+        // an institution hangs off an author, so it brings its authors
+        out.add("authors");
+    }
+    return out;
+}
+
+/**
+ * The citation mapping without the rules for what is not included.
+ * @param include - what is included
+ * @returns the mapping
+ */
+function citationMapping(include: ReadonlySet<Include>): RestMapping {
+    const { work } = OPENALEX_MAPPING.entities;
+    const keepEdge = (type: string): boolean =>
+        type === "cites" ||
+        (type === "authored_by" && include.has("authors")) ||
+        (type === "affiliated_with" && include.has("institutions")) ||
+        (type === "has_topic" && include.has("topics"));
+    const keepEmbedded = (entity: string): boolean => include.has(`${entity}s` as Include);
+    return {
+        ...OPENALEX_MAPPING,
+        entities: {
+            ...OPENALEX_MAPPING.entities,
+            work: {
+                ...work,
+                edges: work.edges?.filter((e) => keepEdge(e.type)),
+                embedded: work.embedded?.filter((e) => keepEmbedded(e.entity)),
+            },
+        },
+    };
+}
+
+/**
+ * The co-authorship mapping: works are containers, every two authors of one work are joined by a
+ * `coauthor` edge, and an author expands through the works they wrote.
+ * @param include - what is included
+ * @param seedIsWork - whether the load starts from a work
+ * @returns the mapping
+ */
+function coauthorshipMapping(include: ReadonlySet<Include>, seedIsWork: boolean): RestMapping {
+    const institutions = include.has("institutions");
+    return {
+        ...OPENALEX_MAPPING,
+        load: seedIsWork ? "by-id" : "authored",
+        seed: undefined,
+        expand: "authored",
+        hydrate: "authors-by-id",
+        directed: false,
+        entities: {
+            ...OPENALEX_MAPPING.entities,
+            work: {
+                id: "id",
+                // a rule's `from` is read with the record's own prefix
+                stripPrefix: PREFIX,
+                node: false,
+                edges: [
+                    { pairs: "authorships[].author.id", type: "coauthor", entity: "author" },
+                    ...(institutions
+                        ? [
+                              {
+                                  each: "authorships[]",
+                                  from: "author.id",
+                                  to: "institutions[].id",
+                                  type: "affiliated_with",
+                                  entity: "institution",
+                              },
+                          ]
+                        : []),
+                ],
+                embedded: [
+                    { at: "authorships[].author", entity: "author" },
+                    ...(institutions ? [{ at: "authorships[].institutions[]", entity: "institution" }] : []),
+                ],
+            },
+        },
+    };
+}
+
+/**
+ * An OpenAlex id param: `W123` or its URL form, checked so a value cannot add a filter.
+ * @param value - the param
+ * @param name - its name
+ * @param letter - the id's kind letter
+ * @param ctx - the context
+ * @returns the bare id
+ */
+function openalexId(value: unknown, name: string, letter: string, ctx: SourceContext): string {
+    const text = typeof value === "string" ? value.trim().replace(PREFIX, "") : "";
+    if (!new RegExp(String.raw`^${letter}\d+$`, "i").test(text)) {
+        failOn(ctx.report, "E_SOURCE_PARAM", `${name} is not an OpenAlex id`, { name, reason: "format" });
+    }
+    return text.toUpperCase();
+}
+
+/**
+ * The plan of an OpenAlex call.
+ * @param params - the params
+ * @param ctx - the context
+ * @param purpose - what the plan is for
+ * @returns the mapping and values
+ */
+function plan(params: OpenAlexParams, ctx: SourceContext, purpose: RestPurpose): RestPlan {
+    const graph = params.graph ?? "citation";
+    const include = includes(params.include);
+    const hasWork = params.work !== undefined && params.work !== "";
+    const hasAuthor = params.author !== undefined && params.author !== "";
+    if (purpose !== "expand" && hasWork === hasAuthor) {
+        failOn(ctx.report, "E_SOURCE_PARAM", "give exactly one of work and author", {
+            name: "work",
+            reason: hasWork ? "exclusive" : "required",
+        });
+    }
+    let seed = "";
+    if (hasWork) {
+        seed = openalexId(params.work, "work", "W", ctx);
+    } else if (hasAuthor) {
+        seed = openalexId(params.author, "author", "A", ctx);
+    }
+    if (graph === "coauthorship") {
+        return {
+            mapping: coauthorshipMapping(include, hasWork),
+            values: { ids: seed, select: purpose === "validate" ? "id,updated_date" : "id,updated_date,authorships" },
+        };
+    }
+    let select =
+        WORK_FIELDS + (include.has("authors") ? ",authorships" : "") + (include.has("topics") ? ",topics" : "");
+    if (purpose === "validate") {
+        select = "id,updated_date";
+    }
+    const mapping = citationMapping(include);
+    return {
+        mapping: hasWork ? mapping : { ...mapping, load: "authored", seed: undefined },
+        values: { ids: seed, select },
+    };
+}
+
+/** OpenAlex's own ids: a kind letter and digits. */
+const ANY_ID = /^[WAIT]\d+$/;
+
+const methods = restMethods<OpenAlexParams>(plan, false);
+
+/**
+ * The OpenAlex preset (https://openalex.org): `work` (the citation graph around a work, default)
+ * or `author` (the works they wrote), `graph: "coauthorship"` for authors joined by shared works,
+ * and `include` (`"authors"`, `"institutions"`, `"topics"`) for more node kinds. Without a key,
+ * the first page loads and a later one warns W_SOURCE_AUTH with `report.source.next`; pass
+ * `credentials.apiKey({ query: "api_key" }, key)` and continue with the cursor. Experimental.
+ * @category Network sources
+ */
+export const openalexSource: GraphSource<OpenAlexParams> = defineSource<OpenAlexParams>({
+    id: OPENALEX_SOURCE_ID,
+    descriptor: OPENALEX_DESCRIPTOR,
+    ...methods,
+    expand(request, sink, ctx) {
+        for (const id of request.ids) {
+            if (!ANY_ID.test(String(id))) {
+                failOn(ctx.report, "E_SOURCE_PARAM", "an expansion seed is not an OpenAlex id", {
+                    name: "ids",
+                    reason: "format",
+                });
+            }
+        }
+        return methods.expand(request, sink, ctx);
+    },
+});