@graphty/graph-io 0.3.47 → 0.3.49

package/dist/sources-rest.js

@@ -0,0 +1,1377 @@
+import { R as REST_DESCRIPTOR, O as OPENALEX_DESCRIPTOR } from "./chunks/descriptor-DLA5kINx.js";
+import { D, S } from "./chunks/descriptor-DLA5kINx.js";
+import { f as failOn, e as SOURCE_ENTITY_ATTR, g as SOURCE_STUB_ATTR, d as defineSource, R as REST_SOURCE_ID, b as failAlone, o as originOf, w as warnOn, O as OPENALEX_SOURCE_ID } from "./chunks/credentials-DXHx7nwR.js";
+import { b as throwIfAborted } from "./chunks/input-CAxt_aV-.js";
+import { ah as ImportError } from "./chunks/report-CwMfYa9J.js";
+import { s as sleep } from "./chunks/context-C3p0RQxR.js";
+const STEP = /^([^.[\]]+)(\[\])?$/;
+const PARSED = /* @__PURE__ */ new Map();
+function parseSelector(selector) {
+  const cached = PARSED.get(selector);
+  if (cached !== void 0) {
+    return cached;
+  }
+  let steps = [];
+  if (selector !== ".") {
+    for (const part of selector.split(".")) {
+      const m = STEP.exec(part);
+      if (m === null) {
+        steps = null;
+        break;
+      }
+      steps.push({ key: m[1], flatten: m[2] !== void 0 });
+    }
+  }
+  PARSED.set(selector, steps);
+  return steps;
+}
+function select(value, selector) {
+  let out = value === void 0 ? [] : [value];
+  for (const step of parseSelector(selector) ?? []) {
+    const next = [];
+    for (const v of out) {
+      if (typeof v !== "object" || v === null || !Object.hasOwn(v, step.key)) {
+        continue;
+      }
+      const got = v[step.key];
+      if (step.flatten && Array.isArray(got)) {
+        next.push(...got);
+      } else if (got !== void 0 && got !== null) {
+        next.push(got);
+      }
+    }
+    out = next;
+  }
+  return out.filter((v) => v !== null && v !== void 0);
+}
+function selectOne(value, selector) {
+  return select(value, selector)[0];
+}
+function selectRecords(body, selector) {
+  const out = select(body, selector);
+  return out.length === 1 && Array.isArray(out[0]) ? out[0] : out;
+}
+const SLOT = /\{([^{}]*)\}/g;
+const NAME = /^[A-Za-z_][A-Za-z0-9_-]*$/;
+function templateNames(template) {
+  const names = [];
+  for (const m of template.matchAll(SLOT)) {
+    if (!NAME.test(m[1])) {
+      return null;
+    }
+    names.push(m[1]);
+  }
+  return /[{}]/.test(template.replaceAll(SLOT, "")) ? null : names;
+}
+function fill(template, values, encode, report) {
+  return template.replaceAll(SLOT, (_, name) => {
+    const value = values[name];
+    if (value === void 0) {
+      failOn(report, "E_SOURCE_PARAM", `the mapping's template needs ${name}`, { name, reason: "required" });
+    }
+    return encode(value);
+  });
+}
+function pathValue(report) {
+  return (value) => {
+    if (value === "." || value === ".." || value === "") {
+      failOn(report, "E_SOURCE_PARAM", "a path value may not be empty or a dot segment", {
+        name: "path",
+        reason: "path"
+      });
+    }
+    return encodeURIComponent(value);
+  };
+}
+function checkOrigin(url, endpoint, report) {
+  const base = endpoint.pathname.replace(/\/$/, "");
+  if (url.origin !== endpoint.origin || !(url.pathname === base || url.pathname.startsWith(`${base}/`))) {
+    failOn(report, "E_SOURCE_PARAM", "a request left the mapping's endpoint", { name: "url", reason: "origin" });
+  }
+  return url;
+}
+function buildRequest(endpoint, request, values, report) {
+  const path = fill(request.path, values, pathValue(report), report);
+  const url = new URL(`${endpoint.origin}${endpoint.pathname.replace(/\/$/, "")}${path}`);
+  for (const [key, template] of Object.entries(request.query ?? {})) {
+    url.searchParams.set(
+      key,
+      fill(template, values, (v) => v, report)
+    );
+  }
+  checkOrigin(url, endpoint, report);
+  if (request.method !== "POST") {
+    return { url, init: {} };
+  }
+  const body = fillBody(request.body, values, report);
+  return {
+    url,
+    init: {
+      method: "POST",
+      headers: { "content-type": "application/json" },
+      body: JSON.stringify(body ?? null),
+      // a mapping's POST reads; it is a query, so it is as safe to retry as a GET
+      idempotent: true
+    }
+  };
+}
+function fillBody(body, values, report) {
+  if (typeof body === "string") {
+    return fill(body, values, (v) => v, report);
+  }
+  if (Array.isArray(body)) {
+    return body.map((b) => fillBody(b, values, report));
+  }
+  if (typeof body === "object" && body !== null) {
+    return Object.fromEntries(Object.entries(body).map(([k, v]) => [k, fillBody(v, values, report)]));
+  }
+  return body;
+}
+function isObject(v) {
+  return typeof v === "object" && v !== null && !Array.isArray(v);
+}
+function checkSelector(value, at, push, required, index = false) {
+  if (value === void 0 && !required) {
+    return;
+  }
+  if (typeof value !== "string") {
+    push("missing", at);
+  } else if (!(index && value === "#") && parseSelector(value) === null) {
+    push("selector", at);
+  }
+}
+function checkTemplate(value, at, push) {
+  if (typeof value !== "string") {
+    push("missing", at);
+  } else if (templateNames(value) === null) {
+    push("template", at);
+  }
+}
+function checkQuery(query, at, push) {
+  if (query === void 0) {
+    return;
+  }
+  if (!isObject(query)) {
+    push("missing", `${at}.query`);
+    return;
+  }
+  for (const [k, v] of Object.entries(query)) {
+    checkTemplate(v, `${at}.query.${k}`, push);
+  }
+}
+function isBatch(batch) {
+  return isObject(batch) && Number.isInteger(batch.max) && batch.max >= 1 && typeof batch.join === "string";
+}
+function checkRequest(r, at, entities, push) {
+  if (!isObject(r)) {
+    push("missing", at);
+    return;
+  }
+  if (r.method !== void 0 && r.method !== "GET" && r.method !== "POST") {
+    push("missing", `${at}.method`);
+  }
+  checkTemplate(r.path, `${at}.path`, push);
+  if (typeof r.path === "string" && !r.path.startsWith("/")) {
+    push("template", `${at}.path`);
+  }
+  checkQuery(r.query, at, push);
+  checkSelector(r.records, `${at}.records`, push, true);
+  if (typeof r.entity !== "string") {
+    push("missing", `${at}.entity`);
+  } else if (!entities.has(r.entity)) {
+    push("unknown-entity", `${at}.entity`);
+  }
+  if (r.batch !== void 0 && !isBatch(r.batch)) {
+    push("batch", `${at}.batch`);
+  }
+  if (r.paging !== void 0) {
+    checkPaging(r.paging, `${at}.paging`, push);
+  }
+}
+function checkPaging(p, at, push) {
+  const paging = p;
+  if (!isObject(paging)) {
+    push("paging", at);
+    return;
+  }
+  switch (paging.style) {
+    case "cursor":
+      if (typeof paging.param !== "string" || typeof paging.first !== "string") {
+        push("paging", at);
+      }
+      checkSelector(paging.next, `${at}.next`, push, true);
+      break;
+    case "offset": {
+      const size = paging.size;
+      const ok = typeof paging.param === "string" && isObject(size) && typeof size.param === "string" && Number.isInteger(size.value) && size.value > 0 && (paging.unit === void 0 || paging.unit === "items" || paging.unit === "pages") && (paging.base === void 0 || paging.base === 0 || paging.base === 1);
+      if (!ok) {
+        push("paging", at);
+      }
+      break;
+    }
+    case "link-header":
+      if (paging.rel !== void 0 && typeof paging.rel !== "string") {
+        push("paging", at);
+      }
+      break;
+    case "next-url":
+      checkSelector(paging.next, `${at}.next`, push, true);
+      break;
+    default:
+      push("paging", `${at}.style`);
+  }
+}
+function checkEdgeRule(rule, at, entities, push) {
+  if (!isObject(rule)) {
+    push("missing", at);
+    return;
+  }
+  const r = rule;
+  if (typeof r.type !== "string") {
+    push("missing", `${at}.type`);
+  }
+  checkSelector(r.each, `${at}.each`, push, false);
+  if ("pairs" in r) {
+    checkSelector(r.pairs, `${at}.pairs`, push, true);
+  } else {
+    checkSelector(r.to, `${at}.to`, push, true);
+    checkSelector(r.from, `${at}.from`, push, false);
+    if (isObject(r.fields)) {
+      for (const [k, v] of Object.entries(r.fields)) {
+        checkSelector(v, `${at}.fields.${k}`, push, true);
+      }
+    } else if (r.fields !== void 0) {
+      push("missing", `${at}.fields`);
+    }
+  }
+  if (r.entity !== void 0 && (typeof r.entity !== "string" || !entities.has(r.entity))) {
+    push("unknown-entity", `${at}.entity`);
+  }
+}
+function checkEntity(e, at, entities, push) {
+  if (!isObject(e)) {
+    push("missing", at);
+    return;
+  }
+  checkSelector(e.id, `${at}.id`, push, true, true);
+  if (e.stripPrefix !== void 0 && typeof e.stripPrefix !== "string") {
+    push("missing", `${at}.stripPrefix`);
+  }
+  if (e.label !== void 0 && typeof e.label !== "string") {
+    push("missing", `${at}.label`);
+  }
+  if (e.node !== void 0 && e.node) {
+    push("missing", `${at}.node`);
+  }
+  if (isObject(e.fields)) {
+    for (const [k, v] of Object.entries(e.fields)) {
+      checkSelector(v, `${at}.fields.${k}`, push, true, true);
+    }
+  } else if (e.fields !== void 0) {
+    push("missing", `${at}.fields`);
+  }
+  if (Array.isArray(e.edges)) {
+    e.edges.forEach((rule, i) => {
+      checkEdgeRule(rule, `${at}.edges.${String(i)}`, entities, push);
+    });
+  } else if (e.edges !== void 0) {
+    push("missing", `${at}.edges`);
+  }
+  if (Array.isArray(e.embedded)) {
+    e.embedded.forEach((em, i) => {
+      const where = `${at}.embedded.${String(i)}`;
+      if (!isObject(em)) {
+        push("missing", where);
+        return;
+      }
+      checkSelector(em.at, `${where}.at`, push, true);
+      if (typeof em.entity !== "string" || !entities.has(em.entity)) {
+        push("unknown-entity", `${where}.entity`);
+      }
+    });
+  } else if (e.embedded !== void 0) {
+    push("missing", `${at}.embedded`);
+  }
+}
+function checkRequestNames(m, requests, push) {
+  const names = [
+    ["load", m.load],
+    ["hydrate", m.hydrate],
+    ["seed", m.seed]
+  ];
+  if (isObject(m.expand)) {
+    names.push(["expand.in", m.expand.in], ["expand.out", m.expand.out]);
+  } else {
+    names.push(["expand", m.expand]);
+  }
+  for (const [at, name] of names) {
+    if (name === void 0 && at !== "load") {
+      continue;
+    }
+    if (typeof name !== "string") {
+      push("missing", at);
+    } else if (!requests.has(name)) {
+      push("unknown-request", at);
+    }
+  }
+}
+function checkEndpoint(endpoint, push) {
+  let url = null;
+  try {
+    url = new URL(typeof endpoint === "string" ? endpoint : "");
+  } catch {
+  }
+  if (url === null || url.protocol !== "https:" && url.protocol !== "http:" || url.username !== "" || url.password !== "" || url.search !== "" || url.hash !== "") {
+    push("endpoint", "endpoint");
+  }
+}
+function checkRestMapping(mapping) {
+  const issues = [];
+  const push = (code, at) => {
+    issues.push(Object.freeze({ code, at }));
+  };
+  if (!isObject(mapping)) {
+    push("missing", "");
+    return issues;
+  }
+  const m = mapping;
+  if (m.version !== 1) {
+    push("version", "version");
+  }
+  checkEndpoint(m.endpoint, push);
+  const requests = isObject(m.requests) ? m.requests : {};
+  const entities = isObject(m.entities) ? m.entities : {};
+  if (!isObject(m.requests)) {
+    push("missing", "requests");
+  }
+  if (!isObject(m.entities)) {
+    push("missing", "entities");
+  }
+  const entityNames = new Set(Object.keys(entities));
+  for (const [name, r] of Object.entries(requests)) {
+    checkRequest(r, `requests.${name}`, entityNames, push);
+  }
+  for (const [name, e] of Object.entries(entities)) {
+    checkEntity(e, `entities.${name}`, entityNames, push);
+  }
+  checkRequestNames(m, new Set(Object.keys(requests)), push);
+  checkLimits(m, push);
+  checkSelector(m.modified, "modified", push, false);
+  if (m.directed !== void 0 && typeof m.directed !== "boolean") {
+    push("missing", "directed");
+  }
+  return issues;
+}
+function checkLimits(m, push) {
+  for (const [at, value] of [
+    ["rateLimit.requests", m.rateLimit?.requests],
+    ["rateLimit.perMs", m.rateLimit?.perMs]
+  ]) {
+    if (m.rateLimit !== void 0 && !(typeof value === "number" && value > 0)) {
+      push("number", at);
+    }
+  }
+  if (m.budgetHeaders !== void 0 && (typeof m.budgetHeaders.remaining !== "string" || m.budgetHeaders.reset !== void 0 && typeof m.budgetHeaders.reset !== "string")) {
+    push("missing", "budgetHeaders");
+  }
+}
+function firstPosition(paging) {
+  switch (paging?.style) {
+    case "cursor":
+      return paging.first;
+    case "offset":
+      return paging.base ?? 0;
+    default:
+      return null;
+  }
+}
+function pageUrl(url, paging, position, endpoint, report) {
+  if (position === null) {
+    return url;
+  }
+  const out = new URL(url);
+  switch (paging?.style) {
+    case "cursor":
+      out.searchParams.set(paging.param, String(position));
+      return out;
+    case "offset":
+      out.searchParams.set(paging.param, String(position));
+      out.searchParams.set(paging.size.param, String(paging.size.value));
+      return out;
+    default:
+      return checkOrigin(new URL(String(position), url), endpoint, report);
+  }
+}
+function firstPageUrl(url, paging, endpoint, report) {
+  return pageUrl(url, paging, firstPosition(paging), endpoint, report);
+}
+function linkOf(header, rel) {
+  for (const part of header.split(/,(?=\s*<)/)) {
+    const m = /^\s*<([^>]*)>(.*)$/.exec(part);
+    if (m === null) {
+      continue;
+    }
+    const rels = /;\s*rel="?([^";]+)"?/i.exec(m[2]);
+    if (rels?.[1].split(/\s+/).includes(rel)) {
+      return m[1];
+    }
+  }
+  return null;
+}
+function nextLink(rel, page, endpoint, report) {
+  const header = page.headers.get("link");
+  if (header === null) {
+    if (page.runtime === "browser") {
+      failOn(report, "E_SOURCE_RESPONSE", "the Link header cannot be read", {
+        expected: "link-header",
+        contentType: page.headers.get("content-type")
+      });
+    }
+    return null;
+  }
+  const next = linkOf(header, rel);
+  return next === null ? null : checkOrigin(new URL(next, page.url), endpoint, report).href;
+}
+function nextPosition(paging, position, page, endpoint, report) {
+  switch (paging?.style) {
+    case "cursor": {
+      const next = selectOne(page.body, paging.next);
+      return typeof next === "string" && next !== "" || typeof next === "number" ? next : null;
+    }
+    case "offset": {
+      if (page.records < paging.size.value || page.records === 0) {
+        return null;
+      }
+      const step = paging.unit === "pages" ? 1 : paging.size.value;
+      return Number(position) + step;
+    }
+    case "link-header":
+      return nextLink(paging.rel ?? "next", page, endpoint, report);
+    case "next-url": {
+      const next = selectOne(page.body, paging.next);
+      return typeof next === "string" && next !== "" ? checkOrigin(new URL(next, page.url), endpoint, report).href : null;
+    }
+    default:
+      return null;
+  }
+}
+function newState(known = []) {
+  return { full: /* @__PURE__ */ new Set(), stubs: /* @__PURE__ */ new Set(), known: new Set(known), modified: null };
+}
+function idText(value, entity) {
+  if (typeof value !== "string" && typeof value !== "number") {
+    return null;
+  }
+  const text = String(value);
+  const prefix = entity?.stripPrefix;
+  return prefix !== void 0 && prefix !== "" && text.startsWith(prefix) ? text.slice(prefix.length) : text;
+}
+function fieldValue(record, selector, index) {
+  if (selector === "#") {
+    return index;
+  }
+  return selector.includes("[]") ? select(record, selector) : selectOne(record, selector);
+}
+class PageWriter {
+  /**
+   * Create one.
+   * @param mapping - the mapping
+   * @param state - the load's state
+   */
+  constructor(mapping, state) {
+    this.mapping = mapping;
+    this.state = state;
+    this.out = { nodes: [], edges: [] };
+    this.pending = [];
+  }
+  /**
+   * Add the records of an entity, and their embedded records.
+   * @param records - the records
+   * @param entityName - their entity
+   */
+  addRecords(records, entityName) {
+    const entity = this.mapping.entities[entityName];
+    records.forEach((record, index) => {
+      const id = entity.node === false ? null : idText(fieldValue(record, entity.id, index), entity);
+      if (entity.node !== false) {
+        if (id === null) {
+          return;
+        }
+        this.addNode(record, index, id, entityName, entity);
+      }
+      const modified = this.mapping.modified === void 0 ? void 0 : selectOne(record, this.mapping.modified);
+      if (typeof modified === "string" && (this.state.modified === null || modified > this.state.modified)) {
+        this.state.modified = modified;
+      }
+      this.pending.push({ record, entity: entityName, id });
+      for (const em of entity.embedded ?? []) {
+        this.addRecords(select(record, em.at), em.entity);
+      }
+    });
+  }
+  /**
+   * Write a full record.
+   * @param record - the record
+   * @param index - its position
+   * @param id - its id
+   * @param entityName - its entity
+   * @param entity - the entity
+   */
+  addNode(record, index, id, entityName, entity) {
+    const attrs = { labels: [entity.label ?? entityName] };
+    if (entity.fields === void 0) {
+      for (const [k, v] of Object.entries(record)) {
+        if (k !== "id" && k !== "labels" && (typeof v === "string" || typeof v === "number" || typeof v === "boolean")) {
+          attrs[k] = v;
+        }
+      }
+    } else {
+      for (const [name, selector] of Object.entries(entity.fields)) {
+        const v = fieldValue(record, selector, index);
+        if (v !== void 0) {
+          attrs[name] = v;
+        }
+      }
+    }
+    if (this.state.stubs.has(id)) {
+      attrs[SOURCE_STUB_ATTR] = false;
+    }
+    this.state.full.add(id);
+    this.out.nodes.push({ id, ...attrs });
+  }
+  /**
+   * Make sure an edge end exists: a stub when no record of it was written and the caller does not
+   * hold it.
+   * @param id - the id
+   * @param entity - its entity, when known
+   */
+  end(id, entity) {
+    const { full, stubs, known } = this.state;
+    if (full.has(id) || stubs.has(id) || known.has(id)) {
+      return;
+    }
+    stubs.add(id);
+    this.out.nodes.push({
+      id,
+      [SOURCE_STUB_ATTR]: true,
+      ...entity === void 0 ? {} : { [SOURCE_ENTITY_ATTR]: entity }
+    });
+  }
+  /**
+   * Run one edge rule over a record.
+   * @param rule - the rule
+   * @param record - the record
+   * @param entityName - the record's entity
+   * @param recordId - the record's id (null for a container)
+   */
+  runRule(rule, record, entityName, recordId) {
+    const scopes = rule.each === void 0 ? [record] : select(record, rule.each);
+    if ("pairs" in rule) {
+      this.runPairsRule(rule, scopes, entityName);
+    } else {
+      this.runLinkRule(rule, scopes, entityName, recordId);
+    }
+  }
+  /**
+   * Run a pairs rule: an edge between every two distinct ids in each scope.
+   * @param rule - the rule
+   * @param scopes - the scopes it runs over
+   * @param entityName - the record's entity
+   */
+  runPairsRule(rule, scopes, entityName) {
+    const entityOf = rule.entity ?? entityName;
+    const entity = this.mapping.entities[entityOf];
+    for (const scope of scopes) {
+      const ids = [...new Set(select(scope, rule.pairs).map((v) => idText(v, entity)))].filter(
+        (v) => v !== null
+      );
+      for (let i = 0; i < ids.length; i++) {
+        for (let j = i + 1; j < ids.length; j++) {
+          this.end(ids[i], entityOf);
+          this.end(ids[j], entityOf);
+          this.out.edges.push({ source: ids[i], target: ids[j], type: rule.type });
+        }
+      }
+    }
+  }
+  /**
+   * Run a from/to rule: an edge from the record (or a selected id) to each selected id.
+   * @param rule - the rule
+   * @param scopes - the scopes it runs over
+   * @param entityName - the record's entity
+   * @param recordId - the record's id (null for a container)
+   */
+  runLinkRule(rule, scopes, entityName, recordId) {
+    const own = this.mapping.entities[entityName];
+    const targetEntity = rule.entity === void 0 ? own : this.mapping.entities[rule.entity];
+    for (const scope of scopes) {
+      const from = rule.from === void 0 ? recordId : idText(selectOne(scope, rule.from), own);
+      if (from === null) {
+        continue;
+      }
+      const tos = select(scope, rule.to).map((t) => idText(t, targetEntity)).filter((v) => v !== null);
+      for (const to of tos) {
+        this.end(from, rule.from === void 0 ? entityName : void 0);
+        this.end(to, rule.entity);
+        this.out.edges.push({ ...edgeFields(rule.fields, scope), source: from, target: to, type: rule.type });
+      }
+    }
+  }
+  /**
+   * Run the edge rules of every record added.
+   * @returns the page
+   */
+  finish() {
+    for (const { record, entity, id } of this.pending) {
+      for (const rule of this.mapping.entities[entity].edges ?? []) {
+        this.runRule(rule, record, entity, id);
+      }
+    }
+    this.pending.length = 0;
+    return this.out;
+  }
+}
+function edgeFields(fields, scope) {
+  const out = {};
+  for (const [name, selector] of Object.entries(fields ?? {})) {
+    const v = fieldValue(scope, selector, 0);
+    if (v !== void 0) {
+      out[name] = v;
+    }
+  }
+  return out;
+}
+function toRecords(mapping, groups, state) {
+  const writer = new PageWriter(mapping, state);
+  for (const g of groups) {
+    writer.addRecords(g.records, g.entity);
+  }
+  return writer.finish();
+}
+const AUTH_CODES = /* @__PURE__ */ new Set(["E_SOURCE_AUTH", "E_SOURCE_FORBIDDEN"]);
+function resetDelayMs(reset) {
+  return Math.max(0, reset > 1e9 ? reset * 1e3 - Date.now() : reset * 1e3);
+}
+class RestRun {
+  /**
+   * Create one.
+   * @param plan - the mapping and values
+   * @param ctx - the context
+   * @param paced - whether this run paces itself (the generic source, whose descriptor cannot)
+   */
+  constructor(plan2, ctx, paced) {
+    this.plan = plan2;
+    this.ctx = ctx;
+    this.paced = paced;
+    this.facts = {};
+    this.last = 0;
+    this.endpoint = new URL(plan2.mapping.endpoint);
+    if (ctx.endpoint !== null && originOf(ctx.endpoint) !== this.endpoint.origin) {
+      failOn(ctx.report, "E_SOURCE_PARAM", "the endpoint is not the mapping's", {
+        name: "endpoint",
+        reason: "origin"
+      });
+    }
+  }
+  /** Wait out the mapping's own rate limit (the generic source only). */
+  async pace() {
+    const rate = this.plan.mapping.rateLimit;
+    if (!this.paced || rate === void 0) {
+      return;
+    }
+    const wait = this.last + rate.perMs / rate.requests - Date.now();
+    if (wait > 0) {
+      await sleep(wait, this.ctx.signal);
+    }
+    this.last = Date.now();
+  }
+  /**
+   * Record the budget headers of a response.
+   * @param headers - the response headers
+   */
+  noteBudget(headers) {
+    const names = this.plan.mapping.budgetHeaders;
+    const remaining = names === void 0 ? null : headers.get(names.remaining);
+    if (names === void 0 || remaining === null || !Number.isFinite(Number(remaining))) {
+      return;
+    }
+    const reset = names.reset === void 0 ? null : Number(headers.get(names.reset) ?? Number.NaN);
+    const resetMs = reset === null || !Number.isFinite(reset) ? void 0 : resetDelayMs(reset);
+    this.facts.budget = { remaining: Number(remaining), ...resetMs === void 0 ? {} : { resetMs } };
+  }
+  /**
+   * Read one page of a request.
+   * @param name - the request name
+   * @param values - the template values
+   * @param position - the page's position, or undefined for the first page
+   * @param headers - extra request headers (validation)
+   * @returns the page
+   */
+  async read(name, values, position, headers) {
+    const request = this.plan.mapping.requests[name];
+    const { report } = this.ctx;
+    const built = buildRequest(this.endpoint, request, values, report);
+    const url = position === void 0 ? firstPageUrl(built.url, request.paging, this.endpoint, report) : pageUrl(built.url, request.paging, position, this.endpoint, report);
+    await this.pace();
+    const response = await this.ctx.fetch(url, {
+      ...built.init,
+      headers: { accept: "application/json", ...built.init.headers, ...headers },
+      allowStatus: [304]
+    });
+    this.noteBudget(response.headers);
+    if (response.status === 304) {
+      return { records: [], next: null, headers: response.headers, status: 304 };
+    }
+    const text = await response.text();
+    let body;
+    try {
+      body = JSON.parse(text);
+    } catch {
+      failOn(report, "E_SOURCE_RESPONSE", "the response is not JSON", {
+        expected: "json",
+        contentType: response.headers.get("content-type")
+      });
+    }
+    const records = selectRecords(body, request.records);
+    const here = position ?? null;
+    const next = nextPosition(
+      request.paging,
+      position ?? (request.paging?.style === "offset" ? request.paging.base ?? 0 : here),
+      { body, headers: response.headers, url, records: records.length, runtime: this.ctx.runtime },
+      this.endpoint,
+      report
+    );
+    return { records, next, headers: response.headers, status: response.status };
+  }
+  /**
+   * Record what the facts say so far, so the report carries them even when a later page fails.
+   * @param state - the load's record state
+   * @param next - where to resume, for an expansion
+   * @returns the report so far
+   */
+  finish(state, next = null) {
+    if (state?.modified !== null && state?.modified !== void 0) {
+      this.facts.validators = { ...this.facts.validators, modified: state.modified };
+    }
+    return this.ctx.finish({ ...this.facts, next });
+  }
+  /**
+   * Note the HTTP validators of a load's first response.
+   * @param headers - its headers
+   */
+  noteValidators(headers) {
+    const etag = headers.get("etag");
+    const lastModified = headers.get("last-modified");
+    this.facts.validators = {
+      ...this.facts.validators,
+      ...etag === null ? {} : { etag },
+      ...lastModified === null ? {} : { lastModified }
+    };
+  }
+  /**
+   * The graph facts of a first page.
+   * @returns the meta
+   */
+  meta() {
+    const { directed } = this.plan.mapping;
+    return directed === void 0 ? void 0 : { directed };
+  }
+}
+function stringValues(params, skip) {
+  const out = {};
+  for (const [k, v] of Object.entries(params)) {
+    if (k !== skip && (typeof v === "string" || typeof v === "number" || typeof v === "boolean")) {
+      out[k] = String(v);
+    }
+  }
+  return out;
+}
+async function* loadPages(run) {
+  const { mapping, values } = run.plan;
+  const { cursor } = run.ctx;
+  if (cursor !== null && cursor.r !== mapping.load) {
+    failOn(run.ctx.report, "E_SOURCE_PARAM", "the cursor is not this mapping's", {
+      name: "cursor",
+      reason: "stale-cursor"
+    });
+  }
+  const state = newState();
+  const groups = [];
+  if (cursor === null && mapping.seed !== void 0) {
+    const seed = await run.read(mapping.seed, values, void 0);
+    groups.push({ records: seed.records, entity: mapping.requests[mapping.seed].entity });
+  }
+  const { entity } = mapping.requests[mapping.load];
+  let position = cursor === null ? void 0 : cursor.at;
+  let first = true;
+  for (; ; ) {
+    const page = await run.read(mapping.load, values, position);
+    if (first && cursor === null) {
+      run.noteValidators(page.headers);
+    }
+    const out = toRecords(mapping, [...groups.splice(0), { records: page.records, entity }], state);
+    run.finish(state);
+    const next = page.next === null ? null : { r: mapping.load, at: page.next };
+    yield { nodes: out.nodes, edges: out.edges, next, meta: first ? run.meta() : void 0 };
+    if (page.next === null) {
+      return;
+    }
+    first = false;
+    position = page.next;
+  }
+}
+function writePage(out, sink, ctx, types) {
+  const { report } = ctx;
+  let n = 0;
+  for (const { id, ...attrs } of out.nodes) {
+    if (n++ % 64 === 0) {
+      throwIfAborted(ctx.signal);
+    }
+    try {
+      sink.addNodeRecord(id, attrs);
+      report.counts.nodes++;
+    } catch (err) {
+      report.recordError(err, { element: String(id) });
+      report.counts.skippedNodes++;
+    }
+  }
+  for (const { source, target, ...attrs } of out.edges) {
+    if (n++ % 64 === 0) {
+      throwIfAborted(ctx.signal);
+    }
+    if (types !== void 0 && !types.has(String(attrs.type))) {
+      continue;
+    }
+    try {
+      sink.addEdgeRecord(source, target, attrs);
+      report.counts.edges++;
+    } catch (err) {
+      report.recordError(err, { element: `${String(source)} -> ${String(target)}` });
+      report.counts.skippedEdges++;
+    }
+  }
+}
+function expandRequests(mapping, request) {
+  if (request.depth === 0) {
+    return mapping.hydrate === void 0 ? [] : [mapping.hydrate];
+  }
+  const { expand } = mapping;
+  if (typeof expand === "string" || expand === void 0) {
+    return expand === void 0 ? [] : [expand];
+  }
+  const direction = request.direction ?? "both";
+  const names = [direction !== "out" ? expand.in : void 0, direction !== "in" ? expand.out : void 0];
+  return [...new Set(names.filter((n) => n !== void 0))];
+}
+function batches(ids, max) {
+  const out = [];
+  for (let i = 0; i < ids.length; i += max) {
+    out.push(ids.slice(i, i + max));
+  }
+  return out;
+}
+async function expandInto(run, request, sink) {
+  return new Expansion(run, request, sink).expand();
+}
+class Expansion {
+  /**
+   * Check the expansion and set the sink's direction.
+   * @param run - the call
+   * @param request - the seeds and how far to go
+   * @param sink - where to write
+   */
+  constructor(run, request, sink) {
+    this.run = run;
+    this.request = request;
+    this.sink = sink;
+    this.added = 0;
+    this.wrote = false;
+    const { ctx } = run;
+    this.mapping = run.plan.mapping;
+    this.names = expandRequests(this.mapping, request);
+    if (this.names.length === 0) {
+      failOn(ctx.report, "E_SOURCE_UNSUPPORTED", "the mapping has no request for this expansion", {
+        feature: request.depth === 0 ? "hydrate" : "expand"
+      });
+    }
+    if (request.ids.length === 0) {
+      failOn(ctx.report, "E_SOURCE_PARAM", "an expansion needs seed ids", { name: "ids", reason: "required" });
+    }
+    this.state = newState([...request.ids.map(String), ...(request.exclude ?? []).map(String)]);
+    this.types = request.relationshipTypes === void 0 ? void 0 : new Set(request.relationshipTypes);
+    if (this.mapping.directed !== void 0 && sink.edgeCount === 0) {
+      sink.setDirected(this.mapping.directed);
+    }
+  }
+  /**
+   * Run every hop.
+   * @returns the report
+   */
+  async expand() {
+    let frontier = this.request.ids.map(String);
+    const depth = Math.max(1, this.request.depth ?? 1);
+    for (let hop = 1; hop <= depth && frontier.length > 0; hop++) {
+      const reached = [];
+      const stop = await this.hop(hop, frontier, reached);
+      if (stop !== null) {
+        return stop;
+      }
+      frontier = [...new Set(reached)];
+    }
+    return this.run.finish(this.state, null);
+  }
+  /**
+   * Run one hop: every request over every batch of the frontier.
+   * @param hop - the hop, from 1
+   * @param frontier - the ids this hop starts from
+   * @param reached - collects the ids this hop adds
+   * @returns the report when the expansion stopped, else null
+   */
+  async hop(hop, frontier, reached) {
+    for (const [x, name] of this.names.entries()) {
+      const max = this.mapping.requests[name].batch?.max ?? 1;
+      for (const [b, batch] of batches(frontier, max).entries()) {
+        if (this.skipped({ hop, x, b })) {
+          continue;
+        }
+        const stop = await this.readBatch({ hop, x, b }, name, batch, frontier, reached);
+        if (stop !== null) {
+          return stop;
+        }
+      }
+    }
+    return null;
+  }
+  /**
+   * Whether a resumed first hop already read this batch.
+   * @param at - the batch
+   * @returns true to skip it
+   */
+  skipped(at) {
+    const { hop, x, b } = at;
+    const resume = this.run.ctx.cursor;
+    if (hop !== 1 || resume === null) {
+      return false;
+    }
+    const rx = Number(resume.x);
+    return x < rx || x === rx && b < Number(resume.b);
+  }
+  /**
+   * Where a batch starts: the recorded position for the batch a first hop stopped in.
+   * @param at - the batch
+   * @returns the position, or undefined for the first page
+   */
+  startPosition(at) {
+    const { hop, x, b } = at;
+    const resume = this.run.ctx.cursor;
+    return hop === 1 && resume !== null && x === Number(resume.x) && b === Number(resume.b) ? resume.at ?? void 0 : void 0;
+  }
+  /**
+   * Read one batch page by page.
+   * @param at - the batch
+   * @param name - the request
+   * @param batch - its ids
+   * @param frontier - the hop's ids
+   * @param reached - collects the ids it adds
+   * @returns the report when the expansion stopped, else null
+   */
+  async readBatch(at, name, batch, frontier, reached) {
+    const r = this.mapping.requests[name];
+    const own = { ...this.run.plan.values, id: batch[0], ids: batch.join(r.batch?.join ?? ",") };
+    let position = this.startPosition(at);
+    for (; ; ) {
+      let page;
+      try {
+        page = await this.run.read(name, own, position);
+      } catch (err) {
+        return this.authStop(err, at.hop === 1 ? { x: at.x, b: at.b, at: position ?? null } : null);
+      }
+      this.writeNew(page, r.entity, reached);
+      const stop = this.capStop(at, page, frontier);
+      if (stop !== null || page.next === null) {
+        return stop;
+      }
+      position = page.next;
+    }
+  }
+  /**
+   * Keep what was written when a later page needs a credential; rethrow anything else.
+   * @param err - what the page threw
+   * @param cursor - where to resume
+   * @returns the report
+   */
+  authStop(err, cursor) {
+    if (!this.wrote || !(err instanceof ImportError) || !AUTH_CODES.has(String(err.issue?.code))) {
+      throw err;
+    }
+    const p = err.issue?.params ?? {};
+    warnOn(this.run.ctx.report, "W_SOURCE_AUTH", "a later page needs a credential; the pages before it are kept", {
+      origin: typeof p.origin === "string" ? p.origin : this.run.endpoint.origin,
+      status: typeof p.status === "number" ? p.status : null
+    });
+    return this.run.finish(this.state, cursor);
+  }
+  /**
+   * Write a page and collect the nodes it added.
+   * @param page - the page
+   * @param entity - its records' entity
+   * @param reached - collects the ids it adds
+   */
+  writeNew(page, entity, reached) {
+    const { state } = this;
+    const before = /* @__PURE__ */ new Set([...state.full, ...state.stubs]);
+    const out = toRecords(this.mapping, [{ records: page.records, entity }], state);
+    writePage(out, this.sink, this.run.ctx, this.types);
+    this.wrote = true;
+    for (const node of out.nodes) {
+      const id = String(node.id);
+      if (!before.has(id) && !state.known.has(id)) {
+        reached.push(id);
+        this.added++;
+      }
+    }
+  }
+  /**
+   * Stop at the expansion's limit or the load's caps.
+   * @param at - the batch
+   * @param page - the page just written
+   * @param frontier - the hop's ids
+   * @returns the report when a cap was reached, else null
+   */
+  capStop(at, page, frontier) {
+    const { hop, x, b } = at;
+    const { ctx } = this.run;
+    const limited = this.request.limit !== void 0 && this.added >= this.request.limit;
+    if (limited) {
+      warnOn(ctx.report, "W_SOURCE_LIMIT", "the expansion reached its limit", {
+        limit: "limit",
+        value: this.added
+      });
+    }
+    if (!limited && !ctx.limits.reached()) {
+      return null;
+    }
+    const more = page.next === null ? null : { x, b, at: page.next };
+    return this.run.finish(
+      this.state,
+      hop === 1 ? more ?? nextBatch(x, b, this.names, frontier, this.mapping) : null
+    );
+  }
+}
+function nextBatch(x, b, names, frontier, mapping) {
+  const count = Math.ceil(frontier.length / (mapping.requests[names[x]].batch?.max ?? 1));
+  if (b + 1 < count) {
+    return { x, b: b + 1, at: null };
+  }
+  return x + 1 < names.length ? { x: x + 1, b: 0, at: null } : null;
+}
+async function validateRun(run, seen) {
+  const { mapping, values } = run.plan;
+  const headers = {};
+  if (seen.etag !== void 0) {
+    headers["if-none-match"] = seen.etag;
+  }
+  if (seen.lastModified !== void 0) {
+    headers["if-modified-since"] = seen.lastModified;
+  }
+  const state = newState();
+  const groups = [];
+  if (mapping.seed !== void 0) {
+    const seed = await run.read(mapping.seed, values, void 0);
+    groups.push({ records: seed.records, entity: mapping.requests[mapping.seed].entity });
+  }
+  const page = await run.read(mapping.load, values, void 0, headers);
+  if (page.status === 304) {
+    return { status: "current", via: seen.etag === void 0 ? "modified" : "etag", validators: seen };
+  }
+  toRecords(mapping, [...groups, { records: page.records, entity: mapping.requests[mapping.load].entity }], state);
+  run.noteValidators(page.headers);
+  const now = {
+    ...run.facts.validators,
+    ...state.modified === null ? {} : { modified: state.modified }
+  };
+  if (seen.etag !== void 0 && now.etag !== void 0) {
+    return { status: seen.etag === now.etag ? "current" : "changed", via: "etag", validators: now };
+  }
+  if (seen.modified !== void 0 && now.modified !== void 0) {
+    return now.modified > seen.modified ? { status: "changed", via: "modified", changedAt: now.modified, validators: now } : { status: "current", via: "modified", validators: now };
+  }
+  return { status: "unverifiable", validators: now };
+}
+function restMethods(plan2, paced) {
+  return {
+    records: (params, ctx) => loadPages(new RestRun(plan2(params, ctx, "load"), ctx, paced)),
+    expand: (request, sink, ctx) => expandInto(new RestRun(plan2(request.params, ctx, "expand"), ctx, paced), request, sink),
+    async probe(params, ctx) {
+      const run = new RestRun(plan2(params, ctx, "probe"), ctx, paced);
+      const page = await run.read(run.plan.mapping.load, run.plan.values, void 0);
+      if (page.records.length === 0 && page.status !== 304) {
+        failOn(ctx.report, "E_SOURCE_NOT_FOUND", "the load request found no records", {
+          what: "records",
+          value: run.plan.mapping.load
+        });
+      }
+      return { ok: true };
+    },
+    validate: (params, seen, ctx) => validateRun(new RestRun(plan2(params, ctx, "validate"), ctx, paced), seen)
+  };
+}
+function checkedMapping(mapping, ctx) {
+  const [issue] = checkRestMapping(mapping);
+  if (issue !== void 0) {
+    failOn(ctx.report, "E_SOURCE_PARAM", `the mapping is invalid at ${issue.at}`, {
+      name: "mapping",
+      reason: issue.code,
+      at: issue.at
+    });
+  }
+  return mapping;
+}
+function apiKeyOf(auth) {
+  if (auth?.kind !== "api-key") {
+    return {};
+  }
+  if (auth.header !== void 0) {
+    return { apiKey: { header: auth.header } };
+  }
+  return auth.query === void 0 ? {} : { apiKey: { query: auth.query } };
+}
+function restSource(mapping, info) {
+  const [issue] = checkRestMapping(mapping);
+  if (issue !== void 0) {
+    failAlone("E_SOURCE_INVALID", `the mapping of ${info.id} is invalid at ${issue.at}`, {
+      id: info.id,
+      reason: "mapping",
+      at: issue.at
+    });
+  }
+  const { id, ...fields } = info;
+  const auth = mapping.auth === void 0 ? [] : [mapping.auth.kind];
+  const descriptor = Object.freeze({
+    plainName: id,
+    kind: "rest",
+    params: mapping.params ?? [],
+    defaultEndpoint: mapping.endpoint,
+    auth,
+    authRequired: auth.length === 0 ? "none" : "optional",
+    ...apiKeyOf(mapping.auth),
+    ...mapping.rateLimit === void 0 ? {} : { rateLimit: mapping.rateLimit },
+    ...fields
+  });
+  return defineSource({
+    id,
+    descriptor,
+    ...restMethods((params) => ({ mapping, values: stringValues(params) }), false)
+  });
+}
+const restMappingSource = defineSource({
+  id: REST_SOURCE_ID,
+  descriptor: REST_DESCRIPTOR,
+  ...restMethods(
+    (params, ctx) => ({ mapping: checkedMapping(params.mapping, ctx), values: stringValues(params, "mapping") }),
+    true
+  )
+});
+const PREFIX = "https://openalex.org/";
+const WORK_FIELDS = "id,display_name,publication_year,cited_by_count,updated_date,referenced_works";
+function works(filter, perPage, paged) {
+  return {
+    path: "/works",
+    query: { filter, "per-page": perPage, select: "{select}" },
+    batch: { max: 50, join: "|" },
+    records: "results",
+    entity: "work",
+    ...paged ? { paging: { style: "cursor", param: "cursor", first: "*", next: "meta.next_cursor" } } : {}
+  };
+}
+const named = { id: "id", stripPrefix: PREFIX, fields: { name: "display_name" } };
+const OPENALEX_MAPPING = Object.freeze({
+  version: 1,
+  endpoint: "https://api.openalex.org",
+  load: "citing",
+  seed: "by-id",
+  expand: { in: "citing", out: "by-id" },
+  hydrate: "by-id",
+  directed: true,
+  modified: "updated_date",
+  auth: { kind: "api-key", query: "api_key" },
+  rateLimit: { requests: 10, perMs: 1e3 },
+  budgetHeaders: { remaining: "x-ratelimit-remaining", reset: "x-ratelimit-reset" },
+  requests: {
+    citing: works("cites:{ids}", "200", true),
+    "by-id": works("openalex:{ids}", "50", false),
+    authored: works("authorships.author.id:{ids}", "200", true),
+    "authors-by-id": {
+      path: "/authors",
+      query: { filter: "openalex:{ids}", "per-page": "50", select: "id,display_name" },
+      batch: { max: 50, join: "|" },
+      records: "results",
+      entity: "author"
+    }
+  },
+  entities: {
+    work: {
+      id: "id",
+      stripPrefix: PREFIX,
+      fields: {
+        title: "display_name",
+        year: "publication_year",
+        citedBy: "cited_by_count",
+        updated: "updated_date"
+      },
+      edges: [
+        { to: "referenced_works[]", type: "cites", entity: "work" },
+        { to: "authorships[].author.id", type: "authored_by", entity: "author" },
+        {
+          each: "authorships[]",
+          from: "author.id",
+          to: "institutions[].id",
+          type: "affiliated_with",
+          entity: "institution"
+        },
+        { to: "topics[].id", type: "has_topic", entity: "topic" }
+      ],
+      embedded: [
+        { at: "authorships[].author", entity: "author" },
+        { at: "authorships[].institutions[]", entity: "institution" },
+        { at: "topics[]", entity: "topic" }
+      ]
+    },
+    author: named,
+    institution: named,
+    topic: named
+  }
+});
+function includes(include) {
+  const list = typeof include === "string" ? include.split(",") : [];
+  if (Array.isArray(include)) {
+    list.push(...include);
+  }
+  const out = new Set(list.map((v) => String(v).trim()).filter((v) => v !== ""));
+  if (out.has("institutions")) {
+    out.add("authors");
+  }
+  return out;
+}
+function citationMapping(include) {
+  const { work } = OPENALEX_MAPPING.entities;
+  const keepEdge = (type) => type === "cites" || type === "authored_by" && include.has("authors") || type === "affiliated_with" && include.has("institutions") || type === "has_topic" && include.has("topics");
+  const keepEmbedded = (entity) => include.has(`${entity}s`);
+  return {
+    ...OPENALEX_MAPPING,
+    entities: {
+      ...OPENALEX_MAPPING.entities,
+      work: {
+        ...work,
+        edges: work.edges?.filter((e) => keepEdge(e.type)),
+        embedded: work.embedded?.filter((e) => keepEmbedded(e.entity))
+      }
+    }
+  };
+}
+function coauthorshipMapping(include, seedIsWork) {
+  const institutions = include.has("institutions");
+  return {
+    ...OPENALEX_MAPPING,
+    load: seedIsWork ? "by-id" : "authored",
+    seed: void 0,
+    expand: "authored",
+    hydrate: "authors-by-id",
+    directed: false,
+    entities: {
+      ...OPENALEX_MAPPING.entities,
+      work: {
+        id: "id",
+        // a rule's `from` is read with the record's own prefix
+        stripPrefix: PREFIX,
+        node: false,
+        edges: [
+          { pairs: "authorships[].author.id", type: "coauthor", entity: "author" },
+          ...institutions ? [
+            {
+              each: "authorships[]",
+              from: "author.id",
+              to: "institutions[].id",
+              type: "affiliated_with",
+              entity: "institution"
+            }
+          ] : []
+        ],
+        embedded: [
+          { at: "authorships[].author", entity: "author" },
+          ...institutions ? [{ at: "authorships[].institutions[]", entity: "institution" }] : []
+        ]
+      }
+    }
+  };
+}
+function openalexId(value, name, letter, ctx) {
+  const text = typeof value === "string" ? value.trim().replace(PREFIX, "") : "";
+  if (!new RegExp(String.raw`^${letter}\d+$`, "i").test(text)) {
+    failOn(ctx.report, "E_SOURCE_PARAM", `${name} is not an OpenAlex id`, { name, reason: "format" });
+  }
+  return text.toUpperCase();
+}
+function plan(params, ctx, purpose) {
+  const graph = params.graph ?? "citation";
+  const include = includes(params.include);
+  const hasWork = params.work !== void 0 && params.work !== "";
+  const hasAuthor = params.author !== void 0 && params.author !== "";
+  if (purpose !== "expand" && hasWork === hasAuthor) {
+    failOn(ctx.report, "E_SOURCE_PARAM", "give exactly one of work and author", {
+      name: "work",
+      reason: hasWork ? "exclusive" : "required"
+    });
+  }
+  let seed = "";
+  if (hasWork) {
+    seed = openalexId(params.work, "work", "W", ctx);
+  } else if (hasAuthor) {
+    seed = openalexId(params.author, "author", "A", ctx);
+  }
+  if (graph === "coauthorship") {
+    return {
+      mapping: coauthorshipMapping(include, hasWork),
+      values: { ids: seed, select: purpose === "validate" ? "id,updated_date" : "id,updated_date,authorships" }
+    };
+  }
+  let select2 = WORK_FIELDS + (include.has("authors") ? ",authorships" : "") + (include.has("topics") ? ",topics" : "");
+  if (purpose === "validate") {
+    select2 = "id,updated_date";
+  }
+  const mapping = citationMapping(include);
+  return {
+    mapping: hasWork ? mapping : { ...mapping, load: "authored", seed: void 0 },
+    values: { ids: seed, select: select2 }
+  };
+}
+const ANY_ID = /^[WAIT]\d+$/;
+const methods = restMethods(plan, false);
+const openalexSource = defineSource({
+  id: OPENALEX_SOURCE_ID,
+  descriptor: OPENALEX_DESCRIPTOR,
+  ...methods,
+  expand(request, sink, ctx) {
+    for (const id of request.ids) {
+      if (!ANY_ID.test(String(id))) {
+        failOn(ctx.report, "E_SOURCE_PARAM", "an expansion seed is not an OpenAlex id", {
+          name: "ids",
+          reason: "format"
+        });
+      }
+    }
+    return methods.expand(request, sink, ctx);
+  }
+});
+export {
+  D as DEPS_DEV_MAPPING,
+  OPENALEX_DESCRIPTOR,
+  OPENALEX_MAPPING,
+  REST_DESCRIPTOR,
+  S as STRING_MAPPING,
+  checkRestMapping,
+  openalexSource,
+  restMappingSource,
+  restSource
+};
+//# sourceMappingURL=sources-rest.js.map