@graphty/graph-io 0.3.47 → 0.3.49

package/src/sources/rest/paging.ts

@@ -0,0 +1,180 @@
+/**
+ * Paging of the REST mapping (design section 13.2): a cursor in the body, an offset, the `Link`
+ * header, or a next URL in the body. A position is a string or a number, so it fits a SourceCursor;
+ * a position that is a URL is checked against the mapping's origin before it is followed.
+ */
+
+import { type ImportReportBuilder } from "../../common/report.js";
+import { failOn } from "../fail.js";
+import { type RestPaging } from "./params.js";
+import { selectOne } from "./selectors.js";
+import { checkOrigin } from "./templates.js";
+
+/** Where a paged request is: the first page (null), a cursor or offset, or a full URL. */
+export type PagePosition = string | number | null;
+
+/**
+ * The position of the first page.
+ * @param paging - the request's paging
+ * @returns the position
+ */
+function firstPosition(paging: RestPaging | undefined): PagePosition {
+    switch (paging?.style) {
+        case "cursor":
+            return paging.first;
+        case "offset":
+            return paging.base ?? 0;
+        default:
+            return null;
+    }
+}
+
+/**
+ * The URL of a page.
+ * @param url - the request's first-page URL
+ * @param paging - the request's paging
+ * @param position - where to read
+ * @param endpoint - the mapping's endpoint, which a full URL must stay on
+ * @param report - the load's report
+ * @returns the page's URL
+ */
+export function pageUrl(
+    url: URL,
+    paging: RestPaging | undefined,
+    position: PagePosition,
+    endpoint: URL,
+    report: ImportReportBuilder,
+): URL {
+    if (position === null) {
+        return url;
+    }
+    const out = new URL(url);
+    switch (paging?.style) {
+        case "cursor":
+            out.searchParams.set(paging.param, String(position));
+            return out;
+        case "offset":
+            out.searchParams.set(paging.param, String(position));
+            out.searchParams.set(paging.size.param, String(paging.size.value));
+            return out;
+        default:
+            // link-header and next-url positions are full URLs from a response (or a recorded cursor)
+            return checkOrigin(new URL(String(position), url), endpoint, report);
+    }
+}
+
+/**
+ * The first page's URL with its paging parameters, so an offset request carries its size.
+ * @param url - the request's URL
+ * @param paging - the request's paging
+ * @param endpoint - the mapping's endpoint
+ * @param report - the load's report
+ * @returns the URL
+ */
+export function firstPageUrl(
+    url: URL,
+    paging: RestPaging | undefined,
+    endpoint: URL,
+    report: ImportReportBuilder,
+): URL {
+    return pageUrl(url, paging, firstPosition(paging), endpoint, report);
+}
+
+/**
+ * The `rel` URL of a Link header.
+ * @param header - the header's value
+ * @param rel - the relation
+ * @returns the URL text, or null
+ */
+function linkOf(header: string, rel: string): string | null {
+    for (const part of header.split(/,(?=\s*<)/)) {
+        const m = /^\s*<([^>]*)>(.*)$/.exec(part);
+        if (m === null) {
+            continue;
+        }
+        const rels = /;\s*rel="?([^";]+)"?/i.exec(m[2]);
+        if (rels?.[1].split(/\s+/).includes(rel)) {
+            return m[1];
+        }
+    }
+    return null;
+}
+
+/** What `nextPosition` reads. */
+interface PageResponse {
+    /** The parsed body. */
+    readonly body: unknown;
+    /** The response headers. */
+    readonly headers: Headers;
+    /** The URL the page was read from. */
+    readonly url: URL;
+    /** How many records the page held. */
+    readonly records: number;
+    /** Where the load runs: a missing Link header in a browser may be one the service hides. */
+    readonly runtime: string;
+}
+
+/**
+ * The next page's URL from the Link header, or null after the last one.
+ * @param rel - the relation that names the next page
+ * @param page - this page's response
+ * @param endpoint - the mapping's endpoint
+ * @param report - the load's report
+ * @returns the next position
+ */
+function nextLink(rel: string, page: PageResponse, endpoint: URL, report: ImportReportBuilder): PagePosition {
+    const header = page.headers.get("link");
+    if (header === null) {
+        if (page.runtime === "browser") {
+            // a browser reads Link only when the service exposes it: never a silent single page
+            failOn(report, "E_SOURCE_RESPONSE", "the Link header cannot be read", {
+                expected: "link-header",
+                contentType: page.headers.get("content-type"),
+            });
+        }
+        return null;
+    }
+    const next = linkOf(header, rel);
+    return next === null ? null : checkOrigin(new URL(next, page.url), endpoint, report).href;
+}
+
+/**
+ * Where the next page is, or null after the last one.
+ * @param paging - the request's paging
+ * @param position - this page's position
+ * @param page - this page's response
+ * @param endpoint - the mapping's endpoint
+ * @param report - the load's report
+ * @returns the next position
+ */
+export function nextPosition(
+    paging: RestPaging | undefined,
+    position: PagePosition,
+    page: PageResponse,
+    endpoint: URL,
+    report: ImportReportBuilder,
+): PagePosition {
+    switch (paging?.style) {
+        case "cursor": {
+            const next = selectOne(page.body, paging.next);
+            return (typeof next === "string" && next !== "") || typeof next === "number" ? next : null;
+        }
+        case "offset": {
+            if (page.records < paging.size.value || page.records === 0) {
+                return null;
+            }
+            const step = paging.unit === "pages" ? 1 : paging.size.value;
+            return Number(position) + step;
+        }
+        case "link-header":
+            return nextLink(paging.rel ?? "next", page, endpoint, report);
+        case "next-url": {
+            const next = selectOne(page.body, paging.next);
+            return typeof next === "string" && next !== ""
+                ? checkOrigin(new URL(next, page.url), endpoint, report).href
+                : null;
+        }
+        default:
+            return null;
+    }
+}

package/src/sources/rest/params.ts

@@ -0,0 +1,243 @@
+/**
+ * The REST mapping format and the parameters of the `rest` source and its `openalex` preset (design
+ * sections 13.2 and 13.3). Experimental: the mapping carries `version: 1`. The REST work may add
+ * optional fields only.
+ */
+
+import { type SourceParamDescriptor } from "../types.js";
+
+/**
+ * How a paged request moves to its next page. Experimental.
+ * @category Network sources
+ */
+export type RestPaging =
+    | {
+          /** The paging style. */
+          readonly style: "cursor";
+          /** The query parameter that carries the position. */
+          readonly param: string;
+          /** The position of the first page. */
+          readonly first: string;
+          /** The selector of the next position in the body. */
+          readonly next: string;
+      }
+    | {
+          /** The paging style. */
+          readonly style: "offset";
+          /** The query parameter that carries the position. */
+          readonly param: string;
+          /** The page size and its query parameter. */
+          readonly size: {
+              /** The query parameter that carries the page size. */
+              readonly param: string;
+              /** The page size. */
+              readonly value: number;
+          };
+          /** Whether the offset counts items or pages. */
+          readonly unit?: "items" | "pages" | undefined;
+          /** The first offset. */
+          readonly base?: 0 | 1 | undefined;
+      }
+    | {
+          /** The paging style. */
+          readonly style: "link-header";
+          /** The link relation of the next page. */
+          readonly rel?: string | undefined;
+      }
+    | {
+          /** The paging style. */
+          readonly style: "next-url";
+          /** The selector of the next position in the body. */
+          readonly next: string;
+      };
+
+/**
+ * One named request of a mapping. Experimental.
+ * @category Network sources
+ */
+export interface RestRequest {
+    /** GET (default) or POST. */
+    readonly method?: "GET" | "POST" | undefined;
+    /** A path template: "/works/{id}". */
+    readonly path: string;
+    /** Query templates: `{ filter: "cites:{work}" }`. */
+    readonly query?: Readonly<Record<string, string>> | undefined;
+    /** A POST body; templates in string leaves. */
+    readonly body?: unknown;
+    /** The selector of the record array, "." for one record. */
+    readonly records: string;
+    /** Which entity the records are. */
+    readonly entity: string;
+    /** How it pages. */
+    readonly paging?: RestPaging | undefined;
+    /** `{ids}` holds up to `max` ids joined by `join`. */
+    readonly batch?:
+        | {
+              /** At most this many ids per request. */
+              readonly max: number;
+              /** What joins the ids. */
+              readonly join: string;
+          }
+        | undefined;
+}
+
+/**
+ * One rule that makes edges from a record. Experimental.
+ * @category Network sources
+ */
+export type RestEdgeRule =
+    | {
+          /** A selector the rule repeats over. */
+          readonly each?: string | undefined;
+          /** The selector of the edge's source; the record's id by default. */
+          readonly from?: string | undefined;
+          /** The selector of the edge's target ids. */
+          readonly to: string;
+          /** The edge type. */
+          readonly type: string;
+          /** The entity the target or record is. */
+          readonly entity?: string | undefined;
+          /** Attribute name to selector. */
+          readonly fields?: Readonly<Record<string, string>> | undefined;
+      }
+    | {
+          /** A selector the rule repeats over. */
+          readonly each?: string | undefined;
+          /** A selector of ids joined pairwise, undirected. */
+          readonly pairs: string;
+          /** The edge type. */
+          readonly type: string;
+          /** The entity the ids are, for its `stripPrefix` and for stubs; the record's own by default. */
+          readonly entity?: string | undefined;
+      };
+
+/**
+ * How records of one kind become nodes and edges. Experimental.
+ * @category Network sources
+ */
+export interface RestEntity {
+    /** The id selector within the record; `#` is the record's position in its array. */
+    readonly id: string;
+    /**
+     * false: the records are containers, not nodes; only their edge rules and embedded records
+     * count (a body holding a node array and an edge array).
+     */
+    readonly node?: false | undefined;
+    /** The node label; the entity's name by default. */
+    readonly label?: string | undefined;
+    /** A prefix stripped from ids ("https://openalex.org/"). */
+    readonly stripPrefix?: string | undefined;
+    /** Attribute name to selector; without it every top-level scalar is kept. */
+    readonly fields?: Readonly<Record<string, string>> | undefined;
+    /** The edge rules. */
+    readonly edges?: readonly RestEdgeRule[] | undefined;
+    /** Nested records that are nodes of their own entity. */
+    readonly embedded?:
+        | readonly {
+              /** The selector of the nested records. */
+              readonly at: string;
+              /** The entity the target or record is. */
+              readonly entity: string;
+          }[]
+        | undefined;
+}
+
+/**
+ * A declarative mapping from a REST API to a graph, as JSON data. Experimental.
+ * @category Network sources
+ */
+export interface RestMapping {
+    /** The format version. */
+    readonly version: 1;
+    /** Every request must stay on this origin. */
+    readonly endpoint: string;
+    /** The named requests. */
+    readonly requests: Readonly<Record<string, RestRequest>>;
+    /** How records become nodes and edges. */
+    readonly entities: Readonly<Record<string, RestEntity>>;
+    /** The request a load runs. */
+    readonly load: string;
+    /** The request an expansion runs per batch of seeds. */
+    readonly expand?:
+        | string
+        | {
+              /** The request for incoming edges. */
+              readonly in?: string | undefined;
+              /** The request for outgoing edges. */
+              readonly out?: string | undefined;
+          }
+        | undefined;
+    /** The request that fetches stub nodes' full records (an expansion of depth 0). */
+    readonly hydrate?: string | undefined;
+    /** A request run once before the load, for the seeds' own records. */
+    readonly seed?: string | undefined;
+    /** Whether the graph is directed; the builder's default when absent. */
+    readonly directed?: boolean | undefined;
+    /**
+     * The selector of a record's modification time (ISO-8601). The newest one loaded becomes
+     * `report.source.validators.modified`, which `validateSource` compares against.
+     */
+    readonly modified?: string | undefined;
+    /** What credential a form should ask for. */
+    readonly auth?:
+        | {
+              /** The credential kind. */
+              readonly kind: "bearer";
+          }
+        | {
+              /** The credential kind. */
+              readonly kind: "api-key";
+              /** The request header the key goes in. */
+              readonly header?: string | undefined;
+              /** The query parameter the key goes in. */
+              readonly query?: string | undefined;
+          }
+        | undefined;
+    /** Paces requests. */
+    readonly rateLimit?:
+        | {
+              /** At most this many requests per period. */
+              readonly requests: number;
+              /** The period, in milliseconds. */
+              readonly perMs: number;
+              /** At most this many requests at once. */
+              readonly concurrent?: number;
+          }
+        | undefined;
+    /** Response headers telling what budget is left. */
+    readonly budgetHeaders?:
+        | {
+              /** What the service says is left. */
+              readonly remaining: string;
+              /** The header naming when the budget resets. */
+              readonly reset?: string | undefined;
+          }
+        | undefined;
+    /** What a form asks for. */
+    readonly params?: readonly SourceParamDescriptor[] | undefined;
+}
+
+/**
+ * The params of the generic `rest` source: the mapping and the values its templates use.
+ * Experimental.
+ * @category Network sources
+ */
+export type RestParams = {
+    /** The mapping. */
+    readonly mapping: RestMapping;
+} & Readonly<Record<string, unknown>>;
+
+/**
+ * The params of the `openalex` preset. Experimental.
+ * @category Network sources
+ */
+export interface OpenAlexParams {
+    /** The seed work ("W2112090702"). */
+    readonly work?: string | undefined;
+    /** The seed author. */
+    readonly author?: string | undefined;
+    /** "citation" (default) or "coauthorship". */
+    readonly graph?: "citation" | "coauthorship" | undefined;
+    /** What else becomes nodes, as a list or a comma-separated string. @defaultValue [] */
+    readonly include?: readonly ("authors" | "institutions" | "topics")[] | string | undefined;
+}