@defold-typescript/library-types 0.22.0 → 0.24.0

package/scripts/parse-markdown-api.ts

@@ -0,0 +1,247 @@
+/**
+ * The markdown ingestion front-end's parser: a third `library-types` corpus
+ * source beside the LuaLS annotations (`parse-luals.ts`) and the typed
+ * `.script_api` (`scriptApiToFixtureJson`). It reads a library's README/`.md`
+ * prose API — the only machine-readable type source Bucket-C libraries ship —
+ * and produces the same ref-doc `doc` shape those two front-ends feed the shared
+ * emitter (`generateModuleDeclaration`), so the markdown path reuses the exact
+ * emit + fidelity machinery.
+ *
+ * Scope is a **flat signature surface**, not deep prose (a PRD non-goal): the
+ * parser lifts one element per `##`- or `###`-level `<receiver>.<fn>(<args>)` API
+ * header and its `**PARAMETERS**` / `**RETURN**` bullet lists. Both levels are
+ * accepted because the corpus is split on the convention (defold-orthographic and
+ * defold-input write `###`; monarch's `README_API.md` writes `##`); h1 and h4 stay
+ * outside the range, since at those levels a dotted-call-shaped line is document
+ * structure rather than a signature. At either level the receiver may be preceded
+ * by a literal `function ` declaration keyword (rendy writes 9 of its 11 headings
+ * that way); no other prefix is accepted, so prose such as `### see mod.fn()` is
+ * still not a signature. Header-only message sections (`<verb>` with
+ * no dotted receiver or parens) and nested option-table bullets are ignored. A
+ * signature row that names a parameter but gives it no `(type)` loud-fails rather
+ * than silently emitting an untyped `any`.
+ */
+
+/** A single ref-doc parameter or return slot. `is_optional` mirrors the
+ * consumer contract's string flag (`"True"`); it is present only when set. */
+export interface MarkdownParam {
+  name: string;
+  doc: string;
+  types: string[];
+  is_optional?: "True";
+}
+
+export interface MarkdownElement {
+  type: "FUNCTION";
+  name: string;
+  description: string;
+  parameters: MarkdownParam[];
+  returnvalues: MarkdownParam[];
+}
+
+/** The ref-doc `doc` shape `generateModuleDeclaration` consumes. `info.namespace`
+ * carries the README's own module prefix; the front-end retargets it to the
+ * pinned namespace before emitting. */
+export interface MarkdownDoc {
+  info: { namespace: string; brief: string; description: string };
+  elements: MarkdownElement[];
+}
+
+// Only the literal `function` keyword is accepted before the receiver — a general
+// `\w+\s+` prefix would make prose like `### see rendy.set(...)` read as a signature.
+const HEADER = /^#{2,3}\s+(?:function\s+)?([A-Za-z_][\w]*)\.([A-Za-z_][\w]*)\((.*)\)\s*$/;
+const PARAM_MARKER = /^\*\*PARAM(?:ETER|ETERS)?\*\*\s*$/;
+const RETURN_MARKER = /^\*\*RETURNS?\*\*\s*$/;
+// A bullet with a backticked name and a required `(type)` group. A named bullet
+// missing the `(type)` group is an unresolvable row (see `parseSlot`).
+const TYPED_BULLET = /^\*\s+`([^`]+)`\s*\(([^)]*)\)\s*-?\s*(.*)$/;
+const NAMED_BULLET = /^\*\s+`([^`]+)`/;
+
+/** Bracketed header arguments are optional; collect their bare names. A bracket
+ * may be escaped (`duration \[, scaler]` — a README authoring artifact so the
+ * upstream renderer does not read `[, scaler]` as a link), may span a comma
+ * (`data [, overwrite]`), and may cover several arguments at once (`[b, c]`),
+ * in which case every argument inside it is optional. Unlike `splitTypes`, a
+ * comma at any depth ends the current argument: in an argument list a comma
+ * always separates arguments, whether or not a bracket group spans it. */
+function bracketedArgs(argList: string): Set<string> {
+  const optional = new Set<string>();
+  let depth = 0;
+  let current = "";
+  let bracketed = false;
+
+  const flush = () => {
+    const arg = current.trim();
+    if (arg.length > 0 && bracketed) optional.add(arg);
+    current = "";
+    bracketed = false;
+  };
+
+  for (const ch of argList.replace(/\\\[/g, "[").replace(/\\\]/g, "]")) {
+    if (ch === "[") {
+      depth++;
+      continue;
+    }
+    if (ch === "]" && depth > 0) {
+      depth--;
+      continue;
+    }
+    if (ch === ",") {
+      flush();
+      continue;
+    }
+    if (depth > 0 && ch.trim().length > 0) bracketed = true;
+    current += ch;
+  }
+  flush();
+  return optional;
+}
+
+/** Split a documented `(type)` group into single tokens. Both `|` and `,` are
+ * union separators — a README is as likely to write `a|b|nil` as `a, b, nil` —
+ * but only at depth 0, so a comma inside a token's own group (`table[k, v]`,
+ * `function(self, dt`) stays part of that token. `<`/`>` are deliberately not
+ * tracked: no group in the corpus uses them, and they need the `=>`
+ * disambiguation the comparator-side splitter carries. */
+function splitTypes(group: string): string[] {
+  const tokens: string[] = [];
+  let depth = 0;
+  let current = "";
+  for (const ch of group) {
+    if (ch === "(" || ch === "[" || ch === "{") depth++;
+    else if ((ch === ")" || ch === "]" || ch === "}") && depth > 0) depth--;
+    if ((ch === "|" || ch === ",") && depth === 0) {
+      tokens.push(current);
+      current = "";
+      continue;
+    }
+    current += ch;
+  }
+  tokens.push(current);
+  return tokens.map((token) => token.trim()).filter((token) => token.length > 0);
+}
+
+/** Parse one `* `name` (type) doc` bullet into a slot, splitting a `a|b|nil` or
+ * `a, b, nil` union into single tokens. Throws naming `fnName` when the bullet
+ * names a parameter but carries no `(type)`. */
+function parseSlot(
+  label: string,
+  fnName: string,
+  line: string,
+  optionalNames: Set<string>,
+): MarkdownParam {
+  const typed = TYPED_BULLET.exec(line);
+  if (typed === null) {
+    const named = NAMED_BULLET.exec(line);
+    const name = named?.[1] ?? line.trim();
+    throw new Error(
+      `parse-markdown-api: ${label}: ${fnName} row for \`${name}\` has no (type) — cannot resolve to a typed param (row: ${line.trim()})`,
+    );
+  }
+  const name = typed[1] as string;
+  const types = splitTypes(typed[2] as string);
+  const slot: MarkdownParam = { name, doc: (typed[3] as string).trim(), types };
+  if (optionalNames.has(name)) slot.is_optional = "True";
+  return slot;
+}
+
+/** Split the README into `##`/`###` `<receiver>.<fn>(...)` sections, ignoring
+ * headers that are not dotted API signatures. */
+function sections(lines: string[]): { header: RegExpExecArray; body: string[] }[] {
+  const starts: number[] = [];
+  lines.forEach((line, index) => {
+    if (HEADER.test(line)) starts.push(index);
+  });
+  return starts.map((start, i) => {
+    const end = i + 1 < starts.length ? (starts[i + 1] as number) : lines.length;
+    return {
+      header: HEADER.exec(lines[start] as string) as RegExpExecArray,
+      body: lines.slice(start + 1, end),
+    };
+  });
+}
+
+function parseSection(
+  label: string,
+  fnName: string,
+  body: string[],
+  optionalNames: Set<string>,
+): {
+  description: string;
+  parameters: MarkdownParam[];
+  returnvalues: MarkdownParam[];
+} {
+  const descriptionLines: string[] = [];
+  const parameters: MarkdownParam[] = [];
+  const returnvalues: MarkdownParam[] = [];
+  let mode: "none" | "params" | "returns" = "none";
+  let sawMarker = false;
+
+  for (const line of body) {
+    if (PARAM_MARKER.test(line)) {
+      mode = "params";
+      sawMarker = true;
+      continue;
+    }
+    if (RETURN_MARKER.test(line)) {
+      mode = "returns";
+      sawMarker = true;
+      continue;
+    }
+    const isBullet = line.trimStart().startsWith("* ");
+    if (isBullet && mode === "params") {
+      parameters.push(parseSlot(label, fnName, line, optionalNames));
+      continue;
+    }
+    if (isBullet && mode === "returns") {
+      returnvalues.push(parseSlot(label, fnName, line, optionalNames));
+      continue;
+    }
+    // Any non-bullet line closes an open list, so a blank line before an
+    // option-table's `Acceptable values:` bullets stops them being captured.
+    mode = "none";
+    if (!sawMarker && line.trim().length > 0 && !line.startsWith("---")) {
+      descriptionLines.push(line.trim());
+    }
+  }
+
+  return { description: descriptionLines.join(" "), parameters, returnvalues };
+}
+
+/**
+ * `label` names the offending module in every loud-fail message. A library whose
+ * `.md` is usage/tutorial prose (defold-input ships six such modules) yields no
+ * signature section at all; emitting that as an empty namespace would silently
+ * publish a module with no members, so it throws instead.
+ */
+export function parseMarkdownApi(text: string, label = "markdown document"): MarkdownDoc {
+  const elements: MarkdownElement[] = [];
+  const prefixes = new Set<string>();
+
+  for (const { header, body } of sections(text.split("\n"))) {
+    const prefix = header[1] as string;
+    const fn = header[2] as string;
+    const name = `${prefix}.${fn}`;
+    prefixes.add(prefix);
+    const { description, parameters, returnvalues } = parseSection(
+      label,
+      name,
+      body,
+      bracketedArgs(header[3] as string),
+    );
+    elements.push({ type: "FUNCTION", name, description, parameters, returnvalues });
+  }
+
+  if (prefixes.size > 1) {
+    throw new Error(
+      `parse-markdown-api: non-uniform module prefix across headers: ${[...prefixes].sort().join(", ")}`,
+    );
+  }
+  if (elements.length === 0) {
+    throw new Error(
+      `parse-markdown-api: ${label} has no \`##\`/\`###\` \`<receiver>.<fn>(...)\` API signature section — refusing to emit an empty namespace`,
+    );
+  }
+  const namespace = [...prefixes][0] ?? "";
+  return { info: { namespace, brief: "", description: "" }, elements };
+}

package/scripts/parse-openapi-api.ts

@@ -0,0 +1,313 @@
+/**
+ * The OpenAPI/proto ingestion front-end's parser: a fifth `library-types` corpus
+ * source beside the LuaLS annotations (`parse-luals.ts`), the typed `.script_api`
+ * (`scriptApiToFixtureJson`), the README markdown (`parse-markdown-api.ts`), and
+ * the authored `.d.ts` lane. It reads a structured REST swagger (JSON) plus the
+ * realtime `.proto` message list — the machine-readable source a codegen'd Lua
+ * client (nakama-defold) is generated from — and produces the same ref-doc `doc`
+ * shape those front-ends feed the shared emitter (`generateModuleDeclaration`), so
+ * the OpenAPI path reuses the exact emit + fidelity machinery.
+ *
+ * The surface mirrors the client codegen's naming, not the raw wire schema:
+ *
+ * - a swagger `paths` operation (`operationId` `Nakama_AuthenticateCustom`) becomes
+ *   a client RPC function `authenticate_custom` (service prefix stripped, snake);
+ * - a swagger `definitions` object (`apiAccountCustom`) becomes a constructor
+ *   `create_api_account_custom`;
+ * - a realtime proto `message ChannelMessageSend` becomes a realtime message
+ *   constructor `create_channel_message_send_message`.
+ *
+ * Scope is a **flat signature surface**, not deep schema modelling (a PRD
+ * non-goal): swagger scalar types and `$ref`/object/array types collapse to the
+ * emitter's token vocabulary (`string`/`number`/`boolean`/`table`), and proto
+ * scalars/messages likewise. Every parameter and return slot carries at least one
+ * resolvable token — a slot is never left type-empty (the "no silent swallow"
+ * contract the fidelity gate depends on).
+ */
+
+/** A single ref-doc parameter or return slot. `is_optional` mirrors the consumer
+ * contract's string flag (`"True"`); it is present only when set. */
+export interface OpenApiParam {
+  name: string;
+  doc: string;
+  types: string[];
+  is_optional?: "True";
+}
+
+export interface OpenApiElement {
+  type: "FUNCTION";
+  name: string;
+  description: string;
+  parameters: OpenApiParam[];
+  returnvalues: OpenApiParam[];
+}
+
+/** The ref-doc `doc` shape `generateModuleDeclaration` consumes. `info.namespace`
+ * is empty here — the swagger/proto source names no publish alias — and the
+ * front-end retargets the bare element names onto the pinned namespace. */
+export interface OpenApiDoc {
+  info: { namespace: string; brief: string; description: string };
+  elements: OpenApiElement[];
+}
+
+/** camelCase / PascalCase -> snake_case, splitting acronym runs on the trailing
+ * capitalised word (`HTTPStatus` -> `http_status`). */
+function snake(name: string): string {
+  return name
+    .replace(/([a-z0-9])([A-Z])/g, "$1_$2")
+    .replace(/([A-Z]+)([A-Z][a-z])/g, "$1_$2")
+    .toLowerCase();
+}
+
+/** Map a swagger schema type keyword to a resolvable emitter token. Object,
+ * array, and `$ref` shapes all collapse to `table` (a Lua table), the faithful
+ * shape a Lua client marshals them as. */
+function swaggerType(type: string | undefined, hasRef: boolean): string {
+  if (hasRef) return "table";
+  switch (type) {
+    case "integer":
+    case "number":
+      return "number";
+    case "boolean":
+      return "boolean";
+    case "string":
+      return "string";
+    default:
+      // `object`, `array`, and anything unrecognised marshal as a table.
+      return "table";
+  }
+}
+
+interface SwaggerSchema {
+  type?: string;
+  $ref?: string;
+  items?: SwaggerSchema;
+  properties?: Record<string, SwaggerSchema>;
+  required?: string[];
+  additionalProperties?: unknown;
+}
+
+interface SwaggerParameter {
+  name: string;
+  in: string;
+  required?: boolean;
+  type?: string;
+  schema?: SwaggerSchema;
+  description?: string;
+}
+
+interface SwaggerOperation {
+  operationId?: string;
+  summary?: string;
+  description?: string;
+  parameters?: SwaggerParameter[];
+  responses?: Record<string, { schema?: SwaggerSchema; description?: string }>;
+}
+
+interface SwaggerRoot {
+  paths?: Record<string, Record<string, SwaggerOperation>>;
+  definitions?: Record<string, SwaggerSchema>;
+}
+
+/** Strip a leading `Service_` prefix from an operationId, then snake the rest. */
+function operationName(operationId: string): string {
+  const underscore = operationId.indexOf("_");
+  const bare = underscore >= 0 ? operationId.slice(underscore + 1) : operationId;
+  return snake(bare);
+}
+
+function schemaToken(schema: SwaggerSchema): string {
+  return swaggerType(schema.type, typeof schema.$ref === "string");
+}
+
+function swaggerParamSlot(param: SwaggerParameter): OpenApiParam {
+  const token =
+    param.in === "body" && param.schema !== undefined
+      ? schemaToken(param.schema)
+      : swaggerType(param.type, false);
+  const slot: OpenApiParam = {
+    name: param.name,
+    doc: (param.description ?? "").trim(),
+    types: [token],
+  };
+  if (param.required !== true) slot.is_optional = "True";
+  return slot;
+}
+
+function swaggerReturnSlots(op: SwaggerOperation): OpenApiParam[] {
+  const schema = op.responses?.["200"]?.schema;
+  if (schema === undefined) return [];
+  return [{ name: "result", doc: "", types: [schemaToken(schema)] }];
+}
+
+function parseSwaggerOperations(root: SwaggerRoot): OpenApiElement[] {
+  const elements: OpenApiElement[] = [];
+  for (const methods of Object.values(root.paths ?? {})) {
+    for (const op of Object.values(methods)) {
+      if (typeof op.operationId !== "string") continue;
+      elements.push({
+        type: "FUNCTION",
+        name: operationName(op.operationId),
+        description: (op.summary ?? op.description ?? "").trim(),
+        parameters: (op.parameters ?? []).map(swaggerParamSlot),
+        returnvalues: swaggerReturnSlots(op),
+      });
+    }
+  }
+  return elements;
+}
+
+function parseSwaggerDefinitions(root: SwaggerRoot): OpenApiElement[] {
+  const elements: OpenApiElement[] = [];
+  for (const [name, schema] of Object.entries(root.definitions ?? {})) {
+    // Only object definitions become constructors; enum/scalar definitions carry
+    // no properties and would emit a meaningless zero-argument builder.
+    if (schema.type !== "object" || schema.properties === undefined) continue;
+    const required = new Set(schema.required ?? []);
+    const parameters = Object.entries(schema.properties).map(([propName, propSchema]) => {
+      const slot: OpenApiParam = { name: propName, doc: "", types: [schemaToken(propSchema)] };
+      if (!required.has(propName)) slot.is_optional = "True";
+      return slot;
+    });
+    elements.push({
+      type: "FUNCTION",
+      name: `create_${snake(name)}`,
+      description: (root.definitions?.[name] as { description?: string }).description?.trim() ?? "",
+      parameters,
+      returnvalues: [{ name: "result", doc: "", types: ["table"] }],
+    });
+  }
+  return elements;
+}
+
+/** Map a proto field type to a resolvable emitter token. Scalar numerics and
+ * `bytes` collapse per Lua marshalling; message/enum/`map`/`repeated` types are
+ * tables. */
+function protoType(rawType: string): string {
+  const type = rawType.trim();
+  if (type.startsWith("map<") || type.startsWith("repeated ")) return "table";
+  switch (type) {
+    case "double":
+    case "float":
+    case "int32":
+    case "int64":
+    case "uint32":
+    case "uint64":
+    case "sint32":
+    case "sint64":
+    case "fixed32":
+    case "fixed64":
+    case "sfixed32":
+    case "sfixed64":
+      return "number";
+    case "bool":
+      return "boolean";
+    case "string":
+    case "bytes":
+      return "string";
+    default:
+      // A message- or enum-typed field marshals as a table.
+      return "table";
+  }
+}
+
+// A proto field line: an optional `repeated`, a type (`map<...>` allowed), a field
+// name, `=`, a tag, `;`. Reserved lines, `oneof`, options, and nested blocks are
+// ignored by requiring the `= <tag>;` tail.
+const PROTO_FIELD = /^\s*(map<[^>]+>|repeated\s+[\w.]+|[\w.]+)\s+([a-z_]\w*)\s*=\s*\d+\s*;/;
+const PROTO_MESSAGE_OPEN = /^message\s+([A-Za-z_]\w*)\s*\{/;
+// A `oneof` groups alternatives that are the enclosing message's own fields; any
+// other block (`message`/`enum`) owns its fields and must not leak them upward, so
+// non-`oneof` openers default to an isolating `other` frame.
+const PROTO_ONEOF_OPEN = /^\s*oneof\s+[A-Za-z_]\w*\s*\{/;
+
+function parseProtoMessages(protoText: string): OpenApiElement[] {
+  const elements: OpenApiElement[] = [];
+  const lines = protoText.split("\n");
+  let current: { name: string; parameters: OpenApiParam[] } | null = null;
+  // Block-kind frames nested inside the current message, excluding the message
+  // root (which `current !== null` represents). Integer depth cannot tell a
+  // `oneof` alternative (a parent field) from a nested-`message` field (not one);
+  // the frame kinds carry that distinction.
+  let frames: Array<"oneof" | "other"> = [];
+
+  const emit = () => {
+    if (current === null) return;
+    elements.push({
+      type: "FUNCTION",
+      name: `create_${snake(current.name)}_message`,
+      description: "",
+      parameters: current.parameters,
+      returnvalues: [],
+    });
+    current = null;
+    frames = [];
+  };
+
+  // Account one message-body segment (a whole line, or the tail of the open line
+  // after its header brace): count a field only when the frames above the root
+  // are empty or all `oneof`, then apply the segment's braces — a `}` that finds
+  // no nested frame closes the message root.
+  const processBody = (body: string): void => {
+    if (current === null) return;
+    const field = PROTO_FIELD.exec(body);
+    if (field !== null && frames.every((f) => f === "oneof")) {
+      const slot: OpenApiParam = {
+        name: field[2] as string,
+        doc: "",
+        types: [protoType(field[1] as string)],
+      };
+      if (frames.includes("oneof")) slot.is_optional = "True";
+      current.parameters.push(slot);
+    }
+    const openKind: "oneof" | "other" = PROTO_ONEOF_OPEN.test(body) ? "oneof" : "other";
+    for (let i = 0; i < (body.match(/\{/g) ?? []).length; i++) frames.push(openKind);
+    for (let i = 0; i < (body.match(/\}/g) ?? []).length; i++) {
+      if (frames.length > 0) {
+        frames.pop();
+      } else {
+        emit();
+        return;
+      }
+    }
+  };
+
+  for (const line of lines) {
+    if (current === null) {
+      const open = PROTO_MESSAGE_OPEN.exec(line);
+      if (open !== null) {
+        current = { name: open[1] as string, parameters: [] };
+        frames = [];
+        // Process the same line's remainder so `message Ping {}` closes here
+        // instead of swallowing every following top-level message.
+        processBody(line.slice(open[0].length));
+      }
+      continue;
+    }
+    processBody(line);
+  }
+  return elements;
+}
+
+/**
+ * Parse the pinned swagger JSON and realtime `.proto` into the shared ref-doc
+ * `doc` shape. Swagger operations, swagger object definitions, and proto messages
+ * each contribute one element; the first occurrence of a codegen'd name wins so a
+ * definition and a like-named operation never emit a duplicate member.
+ */
+export function parseOpenApi(swaggerText: string, protoText: string): OpenApiDoc {
+  const root = JSON.parse(swaggerText) as SwaggerRoot;
+  const all = [
+    ...parseSwaggerOperations(root),
+    ...parseSwaggerDefinitions(root),
+    ...parseProtoMessages(protoText),
+  ];
+  const seen = new Set<string>();
+  const elements: OpenApiElement[] = [];
+  for (const element of all) {
+    if (seen.has(element.name)) continue;
+    seen.add(element.name);
+    elements.push(element);
+  }
+  return { info: { namespace: "", brief: "", description: "" }, elements };
+}