@xyd-js/openapi2opencli 0.0.0-canary-4287eee-20260630010340

package/__tests__/docs-oracle.test.ts

@@ -0,0 +1,237 @@
+import fs from 'node:fs';
+import path from 'node:path';
+
+import { beforeAll, describe, expect, it } from 'vitest';
+
+import { deferencedOpenAPI } from '@xyd-js/openapi';
+
+import { openapi2opencli } from '../index';
+import {
+  buildOpIndex,
+  commandFlags,
+  type DocsMethodResolved,
+  extractCliCommand,
+  joinDocsMethod,
+  parseOverviewMethods,
+  type SpecOp,
+} from '../oracle/docsOracle';
+
+const ORACLE_DIR = path.join(__dirname, '../oracle');
+const FIXTURES = path.join(__dirname, '../__fixtures__/-2.complex.openai');
+const METHODS = path.join(ORACLE_DIR, 'docs-methods.json');
+const ALLOW = path.join(ORACLE_DIR, 'docs-allowlist.json');
+const FLOOR = path.join(ORACLE_DIR, 'docs-coverage.floor.json');
+const REPORT = path.join(ORACLE_DIR, 'docs-coverage.report.json');
+const OPENAPI = path.join(__dirname, '../../xyd-openapi/__fixtures__/-2.complex.openai/input.yaml');
+const OVERVIEW = 'https://developers.openai.com/api/reference/overview';
+
+const REFRESH = process.env.DOCS_REFRESH === '1';
+const RESEED = process.env.DOCS_RESEED === '1';
+
+async function pool<T, R>(items: T[], concurrency: number, fn: (item: T, i: number) => Promise<R>): Promise<R[]> {
+  const results: R[] = new Array(items.length);
+  let next = 0;
+  await Promise.all(
+    Array.from({ length: concurrency }, async () => {
+      while (next < items.length) {
+        const i = next++;
+        results[i] = await fn(items[i], i);
+      }
+    }),
+  );
+  return results;
+}
+
+// ---- Docs refresh (network; opt-in) --------------------------------------
+describe.skipIf(!REFRESH)('docs oracle refresh (network)', () => {
+  it('fetch overview + 270+ CLI pages → fixtures + docs-methods.json', async () => {
+    const overviewHtml = await fetch(OVERVIEW).then((r) => r.text());
+    const methods = parseOverviewMethods(overviewHtml);
+    expect(methods.length).toBeGreaterThan(250);
+
+    const resolved = await pool(methods, 8, async (m): Promise<DocsMethodResolved | null> => {
+      try {
+        const html = await fetch(m.cliUrl).then((r) => r.text());
+        const cli = extractCliCommand(html);
+        if (!cli) return null;
+        return { ...m, docCommand: cli.command, docFlags: commandFlags(cli.flags) };
+      } catch {
+        return null;
+      }
+    });
+
+    const ok = resolved.filter((m): m is DocsMethodResolved => m !== null);
+    fs.mkdirSync(ORACLE_DIR, { recursive: true });
+    fs.writeFileSync(METHODS, `${JSON.stringify(ok, null, 2)}\n`);
+
+    // Per-method, individually reviewable fixtures.
+    fs.rmSync(FIXTURES, { recursive: true, force: true });
+    for (const m of ok) {
+      const dir = path.join(FIXTURES, m.commandPath.join('__'));
+      fs.mkdirSync(dir, { recursive: true });
+      fs.writeFileSync(
+        path.join(dir, 'expected.json'),
+        `${JSON.stringify(
+          { commandPath: m.commandPath, httpMethod: m.httpMethod, docFlags: m.docFlags, docCommand: m.docCommand, docUrl: m.cliUrl },
+          null,
+          2,
+        )}\n`,
+      );
+    }
+    expect(ok.length).toBeGreaterThan(250);
+  }, 600000);
+});
+
+// ---- Conformance (offline; reads docs-methods.json + the vendored spec) ----
+// Input source: a current openai-openapi vendored alongside the oracle.
+const SPEC = path.join(ORACLE_DIR, 'openai-openapi.yaml');
+const BUILD_INPUTS = process.env.DOCS_BUILD_INPUTS === '1';
+
+const methods: DocsMethodResolved[] = fs.existsSync(METHODS) ? JSON.parse(fs.readFileSync(METHODS, 'utf8')) : [];
+
+interface OurCmd {
+  commandPath: string[];
+  flagNames: Set<string>;
+}
+// Our generated commands, indexed by the source operation `${method} ${path}`.
+const oursByOp = new Map<string, OurCmd>();
+const rawOps = new Map<string, any>();
+let specIndex = new Map<string, SpecOp>();
+
+async function build() {
+  const doc = await deferencedOpenAPI(SPEC);
+  if (!doc) throw new Error(`vendored openai-openapi not found at ${SPEC}`);
+
+  const opcli = openapi2opencli(doc, { cliName: 'openai' });
+  const walk = (cmds: any[], prefix: string[]) => {
+    for (const c of cmds) {
+      const p = [...prefix, c.name];
+      if (c.commands?.length) {
+        walk(c.commands, p);
+      } else if (c['x-openapi']) {
+        const x = c['x-openapi'];
+        const names = [...(c.arguments || []).map((a: any) => a.name), ...(c.options || []).map((o: any) => o.name)];
+        oursByOp.set(`${x.method} ${x.path}`, { commandPath: p, flagNames: new Set<string>(names) });
+      }
+    }
+  };
+  walk(opcli.commands || [], []);
+
+  specIndex = buildOpIndex(doc);
+  for (const [p, item] of Object.entries(doc.paths || {})) {
+    for (const method of ['get', 'post', 'put', 'patch', 'delete']) {
+      const op = (item as any)?.[method];
+      if (op) rawOps.set(`${method} ${p}`, op);
+    }
+  }
+}
+
+const opOf = (m: DocsMethodResolved): SpecOp | null => joinDocsMethod(m.commandPath, m.httpMethod, specIndex);
+const ourOf = (m: DocsMethodResolved): OurCmd | undefined => {
+  const op = opOf(m);
+  return op ? oursByOp.get(`${op.method} ${op.path}`) : undefined;
+};
+const key = (m: DocsMethodResolved) => m.commandPath.join(' ');
+const evalM = (m: DocsMethodResolved) => {
+  const c = ourOf(m);
+  const matched = !!c && c.commandPath.join(' ') === key(m);
+  const flagsOk = matched && m.docFlags.every((f) => c!.flagNames.has(f));
+  return { hasInput: !!opOf(m), matched, flagsOk };
+};
+
+function inputView(op: SpecOp): Record<string, unknown> {
+  const raw = rawOps.get(`${op.method} ${op.path}`) || {};
+  const parameters = (raw.parameters || []).map((p: any) => ({ name: p.name, in: p.in, required: !!p.required }));
+  const body = raw.requestBody?.content || {};
+  const contentType = Object.keys(body)[0];
+  const schema = contentType ? body[contentType].schema : undefined;
+  const requestBody = contentType
+    ? { contentType, required: !!raw.requestBody?.required, properties: Object.keys(schema?.properties || {}) }
+    : undefined;
+  return { httpMethod: op.method, path: op.path, operationId: op.operationId, title: op.title, summary: raw.summary, parameters, requestBody };
+}
+
+describe.skipIf(!methods.length)('docs conformance: openapi → opencli vs OpenAI CLI docs', () => {
+  beforeAll(build, 300000);
+
+  const allow = new Set<string>(
+    fs.existsSync(ALLOW) ? JSON.parse(fs.readFileSync(ALLOW, 'utf8')).knownDivergent || [] : [],
+  );
+
+  // Opt-in: write a per-method input.json (the OpenAPI operation it maps to).
+  it.runIf(BUILD_INPUTS)('write per-method input.json', () => {
+    for (const m of methods) {
+      const dir = path.join(FIXTURES, m.commandPath.join('__'));
+      if (!fs.existsSync(dir)) continue;
+      const op = opOf(m);
+      const input = op
+        ? inputView(op)
+        : { note: 'No matching operation in the vendored openai-openapi (newer/renamed resource).' };
+      fs.writeFileSync(path.join(dir, 'input.json'), `${JSON.stringify(input, null, 2)}\n`);
+    }
+  });
+
+  // Opt-in: regenerate the backlog from whatever currently fails.
+  it.runIf(RESEED)('reseed allowlist (backlog)', () => {
+    const failing = methods.filter((m) => {
+      const { matched, flagsOk } = evalM(m);
+      return !(matched && flagsOk);
+    });
+    fs.writeFileSync(
+      ALLOW,
+      `${JSON.stringify(
+        { note: 'Methods whose generated command/flags do not yet match the OpenAI CLI docs (burn-down backlog). Remove entries as openapi2opencli improves.', knownDivergent: failing.map(key).sort() },
+        null,
+        2,
+      )}\n`,
+    );
+  });
+
+  // One test per documented method (the "hundreds of tests"). Skipped while (re)building.
+  for (const m of methods) {
+    it.skipIf(RESEED || BUILD_INPUTS)(key(m), () => {
+      if (allow.has(key(m))) return; // known divergence — tracked in docs-allowlist.json
+      const { matched, flagsOk } = evalM(m);
+      expect(matched, `command "${key(m)}" not produced from its operation`).toBe(true);
+      expect(flagsOk, `flags missing for "${key(m)}": want ⊇ [${m.docFlags.join(', ')}]`).toBe(true);
+    });
+  }
+
+  // Aggregate coverage + ratchet.
+  it('overall coverage stays above the floor', () => {
+    let withInput = 0;
+    let matched = 0;
+    let flagOk = 0;
+    for (const m of methods) {
+      const r = evalM(m);
+      if (r.hasInput) withInput++;
+      if (r.matched) matched++;
+      if (r.matched && r.flagsOk) flagOk++;
+    }
+    const cmdCov = matched / methods.length;
+    fs.writeFileSync(
+      REPORT,
+      `${JSON.stringify(
+        {
+          methods: methods.length,
+          withInput,
+          commandMatched: matched,
+          fullyMatched: flagOk,
+          commandCoverage: Number(cmdCov.toFixed(4)),
+          fullCoverage: Number((flagOk / methods.length).toFixed(4)),
+          allowlisted: allow.size,
+          missingInput: methods.filter((m) => !evalM(m).hasInput).map(key).slice(0, 30),
+        },
+        null,
+        2,
+      )}\n`,
+    );
+    // eslint-disable-next-line no-console
+    console.log(
+      `[docs] command ${(cmdCov * 100).toFixed(1)}%  full ${((flagOk / methods.length) * 100).toFixed(1)}%  ` +
+        `matched ${matched}/${methods.length}  withInput ${withInput}  allowlisted ${allow.size}`,
+    );
+    const floor = fs.existsSync(FLOOR) ? JSON.parse(fs.readFileSync(FLOOR, 'utf8')).command : 0;
+    expect(cmdCov).toBeGreaterThanOrEqual(floor);
+  });
+});

package/__tests__/openapi2opencli.test.ts

@@ -0,0 +1,31 @@
+import { describe, it } from 'vitest';
+
+import { testFixture } from './utils';
+import type { OpenApi2OpenCliOptions } from '../index';
+
+const tests: { name: string; description: string; options?: OpenApi2OpenCliOptions }[] = [
+  {
+    name: '1.basic',
+    description: 'List + retrieve on a single resource (servers + security binding)',
+  },
+  {
+    name: '2.crud',
+    description: 'Full list/create/retrieve/update/delete with a flattened JSON body',
+  },
+  {
+    name: '3.nested',
+    description: 'Nested sub-resource, multiple path params, and a custom action (archive)',
+  },
+  {
+    name: '4.body-flatten',
+    description: 'Hybrid body strategy: scalars/enums/arrays flatten, nested objects → JSON flag',
+  },
+];
+
+describe('openapi2opencli', () => {
+  for (const t of tests) {
+    it(`[${t.name}]: ${t.description}`, async () => {
+      await testFixture(t.name, t.options);
+    });
+  }
+});

package/__tests__/utils.ts

@@ -0,0 +1,41 @@
+import path from 'node:path';
+import fs from 'node:fs';
+
+import { expect } from 'vitest';
+
+import { deferencedOpenAPI } from '@xyd-js/openapi';
+
+import { openapi2opencli } from '../index';
+import type { OpenApi2OpenCliOptions } from '../index';
+
+// Set REGEN=1 to (re)generate the golden output.json files instead of asserting.
+const REGENERATE = process.env.REGEN === '1';
+
+function fullFixturePath(name: string): string {
+  return path.join(__dirname, '../__fixtures__', name);
+}
+
+function readOutput(name: string) {
+  return JSON.parse(fs.readFileSync(fullFixturePath(name), 'utf8'));
+}
+
+function saveOutput(fixtureName: string, result: unknown) {
+  fs.writeFileSync(fullFixturePath(`${fixtureName}/output.json`), `${JSON.stringify(result, null, 2)}\n`);
+}
+
+/** Run a fixture: deref input.yaml → openapi2opencli → compare against golden output.json. */
+export async function testFixture(fixtureName: string, options?: OpenApi2OpenCliOptions) {
+  const doc = await deferencedOpenAPI(fullFixturePath(`${fixtureName}/input.yaml`));
+  if (!doc) throw new Error(`Failed to load fixture spec: ${fixtureName}/input.yaml`);
+
+  const result = openapi2opencli(doc, options);
+  // Normalize to plain JSON (drops undefined, guarantees serializable parity with the file).
+  const normalized = JSON.parse(JSON.stringify(result));
+
+  if (REGENERATE) {
+    saveOutput(fixtureName, normalized);
+  }
+
+  const expected = readOutput(`${fixtureName}/output.json`);
+  expect(normalized).toEqual(expected);
+}

package/biome.json

@@ -0,0 +1,25 @@
+{
+  "$schema": "https://biomejs.dev/schemas/1.9.3/schema.json",
+  "files": {
+    "ignore": ["dist/**", "node_modules/**", "__fixtures__/**"]
+  },
+  "organizeImports": { "enabled": true },
+  "formatter": {
+    "enabled": true,
+    "indentWidth": 2,
+    "indentStyle": "space",
+    "lineWidth": 120
+  },
+  "javascript": {
+    "formatter": {
+      "quoteStyle": "single",
+      "semicolons": "asNeeded"
+    }
+  },
+  "linter": {
+    "enabled": true,
+    "rules": {
+      "recommended": true
+    }
+  }
+}

package/index.ts

@@ -0,0 +1 @@
+export * from './src';

package/oracle/README.md

@@ -0,0 +1,41 @@
+# Oracle conformance harness
+
+Measures how closely our `OpenAPI → OpenCLI` command surface matches the **real
+[openai-cli](https://github.com/openai/openai-cli)** (the Stainless-generated Go CLI built from
+the same OpenAI spec). The comparison is language-agnostic (commands + flags), so it lives here
+in `openapi2opencli` rather than in the Go generator.
+
+## Files
+
+| File | What |
+|---|---|
+| `parseOpenaiCli.ts` | Tolerant parser of openai-cli `pkg/cmd/*.go` → a canonical `CliSurface` (splits `:`-namespaced resource names, detects `PathParam` flags). |
+| `surface.json` | **Generated**, committed oracle surface (228 commands). CI reads this — no network/Go needed. |
+| `pins.json` | The openai-cli commit the surface was parsed from. |
+| `coverage.floor.json` | The L0 coverage ratchet — the conformance test fails if we drop below it. |
+| `coverage.report.json` | Per-run report (gitignored): per-command diffs + the divergence backlog. |
+
+## Commands
+
+```bash
+# Refresh the oracle from the latest openai-cli (network):
+ORACLE_REFRESH=1 pnpm --filter @xyd-js/openapi2opencli test
+
+# Run conformance (offline; reads surface.json), prints L0/L1 + writes coverage.report.json:
+pnpm --filter @xyd-js/openapi2opencli test
+```
+
+## How "match" is defined (layered)
+
+- **L0** — the set of command paths (`chat completions create`, `models retrieve`, …).
+- **L1** — per-command flag names + required + path-vs-non-path, minus the allowlist.
+
+`src/surface.ts` defines `CliSurface`, `opencliToSurface()`, and `diffSurfaces()`.
+
+## Known divergences (the backlog → raise the floor as these are fixed)
+
+- **Stainless namespacing**: openai-cli prefixes `beta …` (assistants/threads) and `admin organization …`
+  (admin) — these live in Stainless config, not the OpenAPI spec. Future: an `x-cli` grouping override.
+- **Stainless-injected flags**: pagination (`max-items`, allowlisted) and output/streaming flags
+  (`output`, `stream-format`).
+- **Meta commands**: `__complete`, `@completion`, `@manpages` (shell/help tooling).

package/oracle/coverage.floor.json

@@ -0,0 +1,4 @@
+{
+  "l0": 0.18,
+  "note": "L0 command-surface coverage vs openai-cli must not regress below this. Raise it as openapi2opencli improves (e.g. beta/admin Stainless namespacing via x-cli overrides). Regenerate the oracle with ORACLE_REFRESH=1."
+}

package/oracle/docs-allowlist.json

@@ -0,0 +1,201 @@
+{
+  "note": "Methods whose generated command/flags do not yet match the OpenAI CLI docs (burn-down backlog). Remove entries as openapi2opencli improves.",
+  "knownDivergent": [
+    "admin organization admin-api-keys create",
+    "admin organization admin-api-keys delete",
+    "admin organization admin-api-keys list",
+    "admin organization admin-api-keys retrieve",
+    "admin organization audit-logs list",
+    "admin organization certificates activate",
+    "admin organization certificates create",
+    "admin organization certificates deactivate",
+    "admin organization certificates delete",
+    "admin organization certificates list",
+    "admin organization certificates retrieve",
+    "admin organization certificates update",
+    "admin organization data-retention retrieve",
+    "admin organization data-retention update",
+    "admin organization groups create",
+    "admin organization groups delete",
+    "admin organization groups list",
+    "admin organization groups retrieve",
+    "admin organization groups roles create",
+    "admin organization groups roles delete",
+    "admin organization groups roles list",
+    "admin organization groups roles retrieve",
+    "admin organization groups update",
+    "admin organization groups users create",
+    "admin organization groups users delete",
+    "admin organization groups users list",
+    "admin organization groups users retrieve",
+    "admin organization invites create",
+    "admin organization invites delete",
+    "admin organization invites list",
+    "admin organization invites retrieve",
+    "admin organization projects api-keys delete",
+    "admin organization projects api-keys list",
+    "admin organization projects api-keys retrieve",
+    "admin organization projects archive",
+    "admin organization projects certificates activate",
+    "admin organization projects certificates deactivate",
+    "admin organization projects certificates list",
+    "admin organization projects create",
+    "admin organization projects data-retention retrieve",
+    "admin organization projects data-retention update",
+    "admin organization projects groups create",
+    "admin organization projects groups delete",
+    "admin organization projects groups list",
+    "admin organization projects groups retrieve",
+    "admin organization projects groups roles create",
+    "admin organization projects groups roles delete",
+    "admin organization projects groups roles list",
+    "admin organization projects groups roles retrieve",
+    "admin organization projects hosted-tool-permissions retrieve",
+    "admin organization projects hosted-tool-permissions update",
+    "admin organization projects list",
+    "admin organization projects model-permissions delete",
+    "admin organization projects model-permissions retrieve",
+    "admin organization projects model-permissions update",
+    "admin organization projects rate-limits list-rate-limits",
+    "admin organization projects rate-limits update-rate-limit",
+    "admin organization projects retrieve",
+    "admin organization projects roles create",
+    "admin organization projects roles delete",
+    "admin organization projects roles list",
+    "admin organization projects roles retrieve",
+    "admin organization projects roles update",
+    "admin organization projects service-accounts create",
+    "admin organization projects service-accounts delete",
+    "admin organization projects service-accounts list",
+    "admin organization projects service-accounts retrieve",
+    "admin organization projects service-accounts update",
+    "admin organization projects spend-alerts create",
+    "admin organization projects spend-alerts delete",
+    "admin organization projects spend-alerts list",
+    "admin organization projects spend-alerts retrieve",
+    "admin organization projects spend-alerts update",
+    "admin organization projects update",
+    "admin organization projects users create",
+    "admin organization projects users delete",
+    "admin organization projects users list",
+    "admin organization projects users retrieve",
+    "admin organization projects users roles create",
+    "admin organization projects users roles delete",
+    "admin organization projects users roles list",
+    "admin organization projects users roles retrieve",
+    "admin organization projects users update",
+    "admin organization roles create",
+    "admin organization roles delete",
+    "admin organization roles list",
+    "admin organization roles retrieve",
+    "admin organization roles update",
+    "admin organization spend-alerts create",
+    "admin organization spend-alerts delete",
+    "admin organization spend-alerts list",
+    "admin organization spend-alerts retrieve",
+    "admin organization spend-alerts update",
+    "admin organization usage audio-speeches",
+    "admin organization usage audio-transcriptions",
+    "admin organization usage code-interpreter-sessions",
+    "admin organization usage completions",
+    "admin organization usage costs",
+    "admin organization usage embeddings",
+    "admin organization usage file-search-calls",
+    "admin organization usage images",
+    "admin organization usage moderations",
+    "admin organization usage vector-stores",
+    "admin organization usage web-search-calls",
+    "admin organization users delete",
+    "admin organization users list",
+    "admin organization users retrieve",
+    "admin organization users roles create",
+    "admin organization users roles delete",
+    "admin organization users roles list",
+    "admin organization users roles retrieve",
+    "admin organization users update",
+    "batches cancel",
+    "batches retrieve",
+    "beta assistants create",
+    "beta assistants delete",
+    "beta assistants list",
+    "beta assistants retrieve",
+    "beta assistants update",
+    "beta chatkit sessions cancel",
+    "beta chatkit sessions create",
+    "beta chatkit threads delete",
+    "beta chatkit threads list",
+    "beta chatkit threads list-items",
+    "beta chatkit threads retrieve",
+    "beta threads create",
+    "beta threads create-and-run",
+    "beta threads delete",
+    "beta threads messages create",
+    "beta threads messages delete",
+    "beta threads messages list",
+    "beta threads messages retrieve",
+    "beta threads messages update",
+    "beta threads retrieve",
+    "beta threads runs cancel",
+    "beta threads runs create",
+    "beta threads runs list",
+    "beta threads runs retrieve",
+    "beta threads runs steps list",
+    "beta threads runs steps retrieve",
+    "beta threads runs submit-tool-outputs",
+    "beta threads runs update",
+    "beta threads update",
+    "chat completions create",
+    "chat completions retrieve",
+    "chat completions update",
+    "containers files content retrieve",
+    "containers files retrieve",
+    "containers retrieve",
+    "conversations items create",
+    "conversations items retrieve",
+    "conversations update",
+    "files content",
+    "files retrieve",
+    "fine-tuning alpha graders run",
+    "fine-tuning alpha graders validate",
+    "fine-tuning checkpoints permissions create",
+    "fine-tuning checkpoints permissions retrieve",
+    "fine-tuning jobs cancel",
+    "fine-tuning jobs list-events",
+    "fine-tuning jobs pause",
+    "fine-tuning jobs resume",
+    "fine-tuning jobs retrieve",
+    "images create-variation",
+    "images edit",
+    "images generate",
+    "models retrieve",
+    "realtime calls accept",
+    "realtime calls hangup",
+    "realtime calls refer",
+    "realtime calls reject",
+    "responses cancel",
+    "responses compact",
+    "responses input-tokens count",
+    "skills content retrieve",
+    "skills retrieve",
+    "skills update",
+    "skills versions content retrieve",
+    "skills versions retrieve",
+    "uploads cancel",
+    "uploads complete",
+    "vector-stores file-batches cancel",
+    "vector-stores file-batches list-files",
+    "vector-stores files content",
+    "vector-stores files retrieve",
+    "vector-stores files update",
+    "vector-stores retrieve",
+    "vector-stores search",
+    "vector-stores update",
+    "videos create-character",
+    "videos download-content",
+    "videos edit",
+    "videos extend",
+    "videos get-character",
+    "videos remix",
+    "videos retrieve"
+  ]
+}

package/oracle/docs-coverage.floor.json

@@ -0,0 +1,4 @@
+{
+  "command": 0.23,
+  "note": "Per-method command coverage vs OpenAI CLI docs (input-driven, current vendored spec). Biggest remaining lever: admin/beta Stainless namespacing. Refresh docs: DOCS_REFRESH=1; rebuild inputs: DOCS_BUILD_INPUTS=1; reseed backlog: DOCS_RESEED=1."
+}