@defold-typescript/library-types 0.23.0 → 0.24.0

package/scripts/parse-lua-surface.ts

@@ -0,0 +1,187 @@
+/**
+ * A line-oriented reader for the *public surface* of a plain-Lua module — the
+ * member names and parameter names a `local M = {} … return M` file declares.
+ *
+ * `sync-authored-types.ts` gates the authored/forked lane on a forked-vs-generated
+ * identity diff and records that there is no coverage comparison against a primary
+ * source. That holds for *types*: upstream Lua declares none. It does not hold for
+ * *surface*: names and arity are right there in the source, so a fork that drops a
+ * member or a trailing parameter is measurable. This module reads that side; the
+ * comparison lives in `authored-parity.ts`.
+ *
+ * The corpus is a handful of pinned files, so line scanning is what it needs — a
+ * Lua grammar would buy nothing here. Two rules keep it honest:
+ *
+ * - **Column 0 only.** A definition is a member when it starts the line. This is
+ *   what keeps `nakama/util/log.lua`'s `M.log = noop` — reassigned inside three
+ *   different function bodies — out of the surface, and it is why the reader never
+ *   needs to know where a block ends.
+ * - **Loud failure over silent undercount.** A missing `return <name>` or a
+ *   parameter list that does not close on its own line throws, as does a
+ *   `return setmetatable(M, …)` whose metatable carries `__index` or is named by a
+ *   variable whose keys this reader cannot see — either could add members no column-0
+ *   scan will find. The whole point of the instrument is that a dropped member is
+ *   visible; a parser that quietly skips what it cannot read would inflate every
+ *   coverage number it feeds.
+ */
+
+export interface LuaMember {
+  name: string;
+  /** Absent for a non-callable field (`M.SOME_CONSTANT = "X"`), so a field can
+   * never be confused with a zero-arity function. */
+  params?: string[];
+  /** True when the definition ends in `...`, which is not a named parameter. */
+  varargs: boolean;
+  /** The `---` LuaDoc block immediately above the definition, comment markers
+   * stripped, one interior blank line crossed; empty when the block is absent or
+   * opens with a plain `--`. */
+  doc: string;
+  /** True when a comment block *was* there and the `---` rule declined it, which `doc`
+   * alone cannot say — an empty `doc` otherwise means upstream documented nothing. */
+  refusedDoc: boolean;
+}
+
+export interface LuaSurface {
+  moduleLocal: string;
+  members: LuaMember[];
+}
+
+const RETURN_LINE = /^return\s+([A-Za-z_][A-Za-z0-9_]*)\s*;?\s*$/;
+const RETURN_SETMETATABLE = /^return\s+setmetatable\(\s*([A-Za-z_][A-Za-z0-9_]*)\s*,/;
+const IDENTIFIER = "[A-Za-z_][A-Za-z0-9_]*";
+
+/** The metatable argument of a `return setmetatable(<name>, …)`, refused unless its
+ * keys are readable and none of them delegates. `text` runs from the comma to the end
+ * of the source rather than to the matching brace: over-reading can only refuse a file
+ * loudly, where under-reading would accept a delegating module and report its short
+ * surface as complete. */
+function assertNoDelegation(text: string): void {
+  if (text.trimStart().startsWith("{") === false) {
+    throw new Error(
+      "lua surface: the returned `setmetatable` is given a metatable this reader cannot see the keys of, so a delegated member would be invisible — inline the table literal or measure the module another way.",
+    );
+  }
+  if (text.includes("__index")) {
+    throw new Error(
+      "lua surface: the returned metatable carries `__index`, so members reached through it would be invisible to a column-0 scan and the surface would read as complete while short.",
+    );
+  }
+}
+
+function resolveModuleLocal(lines: string[]): string {
+  for (let index = lines.length - 1; index >= 0; index -= 1) {
+    const line = lines[index] as string;
+    const bare = RETURN_LINE.exec(line);
+    if (bare) return bare[1] as string;
+    const wrapped = RETURN_SETMETATABLE.exec(line);
+    if (wrapped) {
+      assertNoDelegation(
+        [line.slice((wrapped[0] as string).length), ...lines.slice(index + 1)].join("\n"),
+      );
+      return wrapped[1] as string;
+    }
+  }
+  throw new Error(
+    "lua surface: the source has no trailing `return <name>`, so the module-local name cannot be derived.",
+  );
+}
+
+/** The parameter names between the parentheses opened at `open` on `line`, with a
+ * `...` tail reported separately. Throws when the list does not close on the same
+ * line rather than dropping the member. */
+function readParams(
+  line: string,
+  open: number,
+  lineNumber: number,
+): { params: string[]; varargs: boolean } {
+  const close = line.indexOf(")", open);
+  if (close === -1) {
+    throw new Error(
+      `lua surface: line ${lineNumber} opens a parameter list that does not close on the same line — ${line.trim()}`,
+    );
+  }
+  const raw = line
+    .slice(open + 1, close)
+    .split(",")
+    .map((part) => part.trim())
+    .filter((part) => part !== "");
+  return { params: raw.filter((part) => part !== "..."), varargs: raw.includes("...") };
+}
+
+/** The contiguous run of comment lines ending at `index - 1`, in source order. */
+function readSegment(lines: string[], index: number): string[] {
+  const segment: string[] = [];
+  for (let cursor = index - 1; cursor >= 0; cursor -= 1) {
+    const line = (lines[cursor] as string).trim();
+    if (!line.startsWith("--") || line.startsWith("--[[")) break;
+    segment.unshift(line);
+  }
+  return segment;
+}
+
+/** The contiguous comment block ending at `index - 1`, kept only when it opens
+ * with `---`. LuaDoc in this corpus is a `---` summary followed by plain `--`
+ * `@param`/`@return` lines, so the marker on the *first* line is what decides.
+ *
+ * A single blank line inside the run is crossed, and only when the segment
+ * attached to the definition opens no block of its own — upstream wraps a long
+ * `@param` list around a blank, which would otherwise truncate the block to its
+ * tag half and lose the summary above. A segment carrying its own `---` is never
+ * absorbed into the one below it, so a blank-separated section header stays out. */
+function readDoc(lines: string[], index: number): Pick<LuaMember, "doc" | "refusedDoc"> {
+  const attached = readSegment(lines, index);
+  if (attached.length === 0) return { doc: "", refusedDoc: false };
+  const above = index - attached.length - 1;
+  const block =
+    attached.some((line) => line.startsWith("---")) ||
+    above < 0 ||
+    (lines[above] as string).trim() !== ""
+      ? attached
+      : [...readSegment(lines, above), ...attached];
+  if (!(block[0] as string).startsWith("---")) return { doc: "", refusedDoc: true };
+  const doc = block
+    .map((line) => line.replace(/^-+\s?/, "").trimEnd())
+    .join("\n")
+    .trim();
+  return { doc, refusedDoc: false };
+}
+
+/**
+ * The public surface of a Lua module source: the module-local name it returns and
+ * every member assigned to it at column 0, in source order. A name defined more
+ * than once keeps its first position and its last definition, matching Lua.
+ */
+export function parseLuaSurface(source: string): LuaSurface {
+  const lines = source.split("\n");
+  const moduleLocal = resolveModuleLocal(lines);
+  const definition = new RegExp(`^function\\s+${moduleLocal}\\.(${IDENTIFIER})\\s*\\(`);
+  const assignment = new RegExp(`^${moduleLocal}\\.(${IDENTIFIER})\\s*=\\s*(.*)$`);
+  const assignedFunction = /^function\s*\(/;
+
+  const members = new Map<string, LuaMember>();
+  const record = (member: LuaMember): void => {
+    members.set(member.name, member);
+  };
+
+  for (const [index, line] of lines.entries()) {
+    const defined = definition.exec(line);
+    if (defined) {
+      const { params, varargs } = readParams(line, defined[0].length - 1, index + 1);
+      record({ name: defined[1] as string, params, varargs, ...readDoc(lines, index) });
+      continue;
+    }
+    const assigned = assignment.exec(line);
+    if (!assigned) continue;
+    const name = assigned[1] as string;
+    const rhs = assigned[2] as string;
+    if (!assignedFunction.test(rhs)) {
+      record({ name, varargs: false, ...readDoc(lines, index) });
+      continue;
+    }
+    const open = line.indexOf("(", line.length - rhs.length);
+    const { params, varargs } = readParams(line, open, index + 1);
+    record({ name, params, varargs, ...readDoc(lines, index) });
+  }
+
+  return { moduleLocal, members: [...members.values()] };
+}

package/scripts/parse-luals.ts

@@ -35,6 +35,7 @@ export interface LibraryInterface {
   // so an interface without overloads carries no key. The emitter renders each as an
   // interface call signature; the mapper maps the token to a `(params): ret` form.
   overloads?: LibraryOverload[];
+  deprecated?: string;
 }
 
 export interface LibraryOverload {
@@ -53,6 +54,10 @@ export interface LibraryMethod {
   // unmarked method carries no key. `local` has no `@field` analogue (LuaLS field
   // scope has no `local`), so a method's visibility widens the field set with it.
   visibility?: LibraryMethodVisibility;
+  // A `---@deprecated` on the pending block. Present exactly when the tag is, with
+  // `""` for the bare form — absence of the key is the sole encoding of "not
+  // deprecated", so a bare tag must never be collapsed away by a truthiness guard.
+  deprecated?: string;
 }
 
 export interface LibraryParam {
@@ -89,6 +94,7 @@ export interface LibraryAlias {
   name: string;
   types: string[];
   doc: string;
+  deprecated?: string;
 }
 
 interface Pending {
@@ -98,6 +104,7 @@ interface Pending {
   generics: LibraryGeneric[];
   overloads: LibraryOverload[];
   visibility?: LibraryMethodVisibility;
+  deprecated?: string;
 }
 
 const emptyPending = (): Pending => ({
@@ -316,7 +323,14 @@ function parseParam(rest: string): LibraryParam {
   return param;
 }
 
-function parseReturn(rest: string): LibraryParam {
+/**
+ * A single `---@return` line may declare several values (`---@return number, number`),
+ * which `readTypeToken`'s comma continuation hands back as one token. Split it on
+ * top-level commas into one entry per value; LuaLS attaches the optional trailing name
+ * and description to the whole line, so only the last segment carries them. The
+ * emitter's `renderReturn` already lowers a multi-entry `returns` to `LuaMultiReturn`.
+ */
+function parseReturn(rest: string): LibraryParam[] {
   const { type, rest: afterType } = readTypeToken(rest);
   const spaceAt = afterType.search(/\s/);
   const head = spaceAt === -1 ? afterType : afterType.slice(0, spaceAt);
@@ -326,7 +340,22 @@ function parseReturn(rest: string): LibraryParam {
     name = head;
     doc = spaceAt === -1 ? "" : afterType.slice(spaceAt).trim();
   }
-  return { name, types: type ? [type] : [], doc, isOptional: false, isVararg: false };
+  const segments = (type ? splitTopLevel(type, ",") : [""])
+    .map((segment) => segment.trim())
+    .filter((segment) => segment !== "");
+  if (segments.length === 0) {
+    return [{ name, types: [], doc, isOptional: false, isVararg: false }];
+  }
+  return segments.map((segment, index) => {
+    const isLast = index === segments.length - 1;
+    return {
+      name: isLast ? name : "",
+      types: [segment],
+      doc: isLast ? doc : "",
+      isOptional: false,
+      isVararg: false,
+    };
+  });
 }
 
 const VISIBILITY_KEYWORDS = new Set<LibraryFieldVisibility>([
@@ -546,6 +575,7 @@ export function parseLualsSource(source: string): LibraryModel {
     params: pending.params,
     returns: pending.returns,
     ...(pending.visibility ? { visibility: pending.visibility } : {}),
+    ...(pending.deprecated !== undefined ? { deprecated: pending.deprecated } : {}),
   });
 
   // Interpret one indented line while a function-local `---@class` block is open. Only
@@ -646,6 +676,7 @@ export function parseLualsSource(source: string): LibraryModel {
           if (pending.doc.length > 0 && iface.brief === "") iface.brief = pending.doc.join("\n");
           if (pending.generics.length > 0) iface.generics = pending.generics;
           if (pending.overloads.length > 0) iface.overloads = pending.overloads;
+          if (pending.deprecated !== undefined) iface.deprecated = pending.deprecated;
           openClass = iface;
           lastOpenedClass = head.name;
           pending = emptyPending();
@@ -664,7 +695,7 @@ export function parseLualsSource(source: string): LibraryModel {
           break;
         }
         case "return": {
-          pending.returns.push(parseReturn(rest));
+          pending.returns.push(...parseReturn(rest));
           break;
         }
         case "generic": {
@@ -684,7 +715,12 @@ export function parseLualsSource(source: string): LibraryModel {
           const spaceAt = rest.search(/\s/);
           const name = spaceAt === -1 ? rest : rest.slice(0, spaceAt);
           const expr = spaceAt === -1 ? "" : rest.slice(spaceAt).trim();
-          aliases.push({ name, types: expr ? [expr] : [], doc: pending.doc.join("\n") });
+          aliases.push({
+            name,
+            types: expr ? [expr] : [],
+            doc: pending.doc.join("\n"),
+            ...(pending.deprecated !== undefined ? { deprecated: pending.deprecated } : {}),
+          });
           pending = emptyPending();
           break;
         }
@@ -698,6 +734,12 @@ export function parseLualsSource(source: string): LibraryModel {
           pending.visibility = tag as LibraryMethodVisibility;
           break;
         }
+        case "deprecated": {
+          // `rest` is already trimmed and is `""` for the bare form, which is the
+          // only form the fixtures actually ship; the text form is carried too.
+          pending.deprecated = rest;
+          break;
+        }
         default:
           // @cast, @type, @diagnostic, ... — outside the Druid subset; recognized as a
           // tag and skipped, never treated as doc.
@@ -793,6 +835,7 @@ export function mergeLibraryModels(models: LibraryModel[]): LibraryModel {
           ...(iface.overloads && iface.overloads.length > 0
             ? { overloads: [...iface.overloads] }
             : {}),
+          ...(iface.deprecated !== undefined ? { deprecated: iface.deprecated } : {}),
         };
         byName.set(iface.name, copy);
         interfaces.push(copy);
@@ -801,6 +844,9 @@ export function mergeLibraryModels(models: LibraryModel[]): LibraryModel {
       existing.fields.push(...iface.fields);
       existing.methods.push(...iface.methods);
       if (!existing.extends && iface.extends) existing.extends = iface.extends;
+      if (existing.deprecated === undefined && iface.deprecated !== undefined) {
+        existing.deprecated = iface.deprecated;
+      }
       if (existing.brief === "" && iface.brief !== "") existing.brief = iface.brief;
       if (existing.generics.length === 0 && iface.generics.length > 0) {
         existing.generics = [...iface.generics];

package/scripts/parse-markdown-api.ts

@@ -0,0 +1,247 @@
+/**
+ * The markdown ingestion front-end's parser: a third `library-types` corpus
+ * source beside the LuaLS annotations (`parse-luals.ts`) and the typed
+ * `.script_api` (`scriptApiToFixtureJson`). It reads a library's README/`.md`
+ * prose API — the only machine-readable type source Bucket-C libraries ship —
+ * and produces the same ref-doc `doc` shape those two front-ends feed the shared
+ * emitter (`generateModuleDeclaration`), so the markdown path reuses the exact
+ * emit + fidelity machinery.
+ *
+ * Scope is a **flat signature surface**, not deep prose (a PRD non-goal): the
+ * parser lifts one element per `##`- or `###`-level `<receiver>.<fn>(<args>)` API
+ * header and its `**PARAMETERS**` / `**RETURN**` bullet lists. Both levels are
+ * accepted because the corpus is split on the convention (defold-orthographic and
+ * defold-input write `###`; monarch's `README_API.md` writes `##`); h1 and h4 stay
+ * outside the range, since at those levels a dotted-call-shaped line is document
+ * structure rather than a signature. At either level the receiver may be preceded
+ * by a literal `function ` declaration keyword (rendy writes 9 of its 11 headings
+ * that way); no other prefix is accepted, so prose such as `### see mod.fn()` is
+ * still not a signature. Header-only message sections (`<verb>` with
+ * no dotted receiver or parens) and nested option-table bullets are ignored. A
+ * signature row that names a parameter but gives it no `(type)` loud-fails rather
+ * than silently emitting an untyped `any`.
+ */
+
+/** A single ref-doc parameter or return slot. `is_optional` mirrors the
+ * consumer contract's string flag (`"True"`); it is present only when set. */
+export interface MarkdownParam {
+  name: string;
+  doc: string;
+  types: string[];
+  is_optional?: "True";
+}
+
+export interface MarkdownElement {
+  type: "FUNCTION";
+  name: string;
+  description: string;
+  parameters: MarkdownParam[];
+  returnvalues: MarkdownParam[];
+}
+
+/** The ref-doc `doc` shape `generateModuleDeclaration` consumes. `info.namespace`
+ * carries the README's own module prefix; the front-end retargets it to the
+ * pinned namespace before emitting. */
+export interface MarkdownDoc {
+  info: { namespace: string; brief: string; description: string };
+  elements: MarkdownElement[];
+}
+
+// Only the literal `function` keyword is accepted before the receiver — a general
+// `\w+\s+` prefix would make prose like `### see rendy.set(...)` read as a signature.
+const HEADER = /^#{2,3}\s+(?:function\s+)?([A-Za-z_][\w]*)\.([A-Za-z_][\w]*)\((.*)\)\s*$/;
+const PARAM_MARKER = /^\*\*PARAM(?:ETER|ETERS)?\*\*\s*$/;
+const RETURN_MARKER = /^\*\*RETURNS?\*\*\s*$/;
+// A bullet with a backticked name and a required `(type)` group. A named bullet
+// missing the `(type)` group is an unresolvable row (see `parseSlot`).
+const TYPED_BULLET = /^\*\s+`([^`]+)`\s*\(([^)]*)\)\s*-?\s*(.*)$/;
+const NAMED_BULLET = /^\*\s+`([^`]+)`/;
+
+/** Bracketed header arguments are optional; collect their bare names. A bracket
+ * may be escaped (`duration \[, scaler]` — a README authoring artifact so the
+ * upstream renderer does not read `[, scaler]` as a link), may span a comma
+ * (`data [, overwrite]`), and may cover several arguments at once (`[b, c]`),
+ * in which case every argument inside it is optional. Unlike `splitTypes`, a
+ * comma at any depth ends the current argument: in an argument list a comma
+ * always separates arguments, whether or not a bracket group spans it. */
+function bracketedArgs(argList: string): Set<string> {
+  const optional = new Set<string>();
+  let depth = 0;
+  let current = "";
+  let bracketed = false;
+
+  const flush = () => {
+    const arg = current.trim();
+    if (arg.length > 0 && bracketed) optional.add(arg);
+    current = "";
+    bracketed = false;
+  };
+
+  for (const ch of argList.replace(/\\\[/g, "[").replace(/\\\]/g, "]")) {
+    if (ch === "[") {
+      depth++;
+      continue;
+    }
+    if (ch === "]" && depth > 0) {
+      depth--;
+      continue;
+    }
+    if (ch === ",") {
+      flush();
+      continue;
+    }
+    if (depth > 0 && ch.trim().length > 0) bracketed = true;
+    current += ch;
+  }
+  flush();
+  return optional;
+}
+
+/** Split a documented `(type)` group into single tokens. Both `|` and `,` are
+ * union separators — a README is as likely to write `a|b|nil` as `a, b, nil` —
+ * but only at depth 0, so a comma inside a token's own group (`table[k, v]`,
+ * `function(self, dt`) stays part of that token. `<`/`>` are deliberately not
+ * tracked: no group in the corpus uses them, and they need the `=>`
+ * disambiguation the comparator-side splitter carries. */
+function splitTypes(group: string): string[] {
+  const tokens: string[] = [];
+  let depth = 0;
+  let current = "";
+  for (const ch of group) {
+    if (ch === "(" || ch === "[" || ch === "{") depth++;
+    else if ((ch === ")" || ch === "]" || ch === "}") && depth > 0) depth--;
+    if ((ch === "|" || ch === ",") && depth === 0) {
+      tokens.push(current);
+      current = "";
+      continue;
+    }
+    current += ch;
+  }
+  tokens.push(current);
+  return tokens.map((token) => token.trim()).filter((token) => token.length > 0);
+}
+
+/** Parse one `* `name` (type) doc` bullet into a slot, splitting a `a|b|nil` or
+ * `a, b, nil` union into single tokens. Throws naming `fnName` when the bullet
+ * names a parameter but carries no `(type)`. */
+function parseSlot(
+  label: string,
+  fnName: string,
+  line: string,
+  optionalNames: Set<string>,
+): MarkdownParam {
+  const typed = TYPED_BULLET.exec(line);
+  if (typed === null) {
+    const named = NAMED_BULLET.exec(line);
+    const name = named?.[1] ?? line.trim();
+    throw new Error(
+      `parse-markdown-api: ${label}: ${fnName} row for \`${name}\` has no (type) — cannot resolve to a typed param (row: ${line.trim()})`,
+    );
+  }
+  const name = typed[1] as string;
+  const types = splitTypes(typed[2] as string);
+  const slot: MarkdownParam = { name, doc: (typed[3] as string).trim(), types };
+  if (optionalNames.has(name)) slot.is_optional = "True";
+  return slot;
+}
+
+/** Split the README into `##`/`###` `<receiver>.<fn>(...)` sections, ignoring
+ * headers that are not dotted API signatures. */
+function sections(lines: string[]): { header: RegExpExecArray; body: string[] }[] {
+  const starts: number[] = [];
+  lines.forEach((line, index) => {
+    if (HEADER.test(line)) starts.push(index);
+  });
+  return starts.map((start, i) => {
+    const end = i + 1 < starts.length ? (starts[i + 1] as number) : lines.length;
+    return {
+      header: HEADER.exec(lines[start] as string) as RegExpExecArray,
+      body: lines.slice(start + 1, end),
+    };
+  });
+}
+
+function parseSection(
+  label: string,
+  fnName: string,
+  body: string[],
+  optionalNames: Set<string>,
+): {
+  description: string;
+  parameters: MarkdownParam[];
+  returnvalues: MarkdownParam[];
+} {
+  const descriptionLines: string[] = [];
+  const parameters: MarkdownParam[] = [];
+  const returnvalues: MarkdownParam[] = [];
+  let mode: "none" | "params" | "returns" = "none";
+  let sawMarker = false;
+
+  for (const line of body) {
+    if (PARAM_MARKER.test(line)) {
+      mode = "params";
+      sawMarker = true;
+      continue;
+    }
+    if (RETURN_MARKER.test(line)) {
+      mode = "returns";
+      sawMarker = true;
+      continue;
+    }
+    const isBullet = line.trimStart().startsWith("* ");
+    if (isBullet && mode === "params") {
+      parameters.push(parseSlot(label, fnName, line, optionalNames));
+      continue;
+    }
+    if (isBullet && mode === "returns") {
+      returnvalues.push(parseSlot(label, fnName, line, optionalNames));
+      continue;
+    }
+    // Any non-bullet line closes an open list, so a blank line before an
+    // option-table's `Acceptable values:` bullets stops them being captured.
+    mode = "none";
+    if (!sawMarker && line.trim().length > 0 && !line.startsWith("---")) {
+      descriptionLines.push(line.trim());
+    }
+  }
+
+  return { description: descriptionLines.join(" "), parameters, returnvalues };
+}
+
+/**
+ * `label` names the offending module in every loud-fail message. A library whose
+ * `.md` is usage/tutorial prose (defold-input ships six such modules) yields no
+ * signature section at all; emitting that as an empty namespace would silently
+ * publish a module with no members, so it throws instead.
+ */
+export function parseMarkdownApi(text: string, label = "markdown document"): MarkdownDoc {
+  const elements: MarkdownElement[] = [];
+  const prefixes = new Set<string>();
+
+  for (const { header, body } of sections(text.split("\n"))) {
+    const prefix = header[1] as string;
+    const fn = header[2] as string;
+    const name = `${prefix}.${fn}`;
+    prefixes.add(prefix);
+    const { description, parameters, returnvalues } = parseSection(
+      label,
+      name,
+      body,
+      bracketedArgs(header[3] as string),
+    );
+    elements.push({ type: "FUNCTION", name, description, parameters, returnvalues });
+  }
+
+  if (prefixes.size > 1) {
+    throw new Error(
+      `parse-markdown-api: non-uniform module prefix across headers: ${[...prefixes].sort().join(", ")}`,
+    );
+  }
+  if (elements.length === 0) {
+    throw new Error(
+      `parse-markdown-api: ${label} has no \`##\`/\`###\` \`<receiver>.<fn>(...)\` API signature section — refusing to emit an empty namespace`,
+    );
+  }
+  const namespace = [...prefixes][0] ?? "";
+  return { info: { namespace, brief: "", description: "" }, elements };
+}