@sungen/driver-api 3.1.2-beta.104 → 3.1.2-beta.106

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sungen/driver-api",
-  "version": "3.1.2-beta.104",
+  "version": "3.1.2-beta.106",
   "description": "Sungen API capability — the @api annotation, API catalog, OpenAPI import + `sungen api import`, and the specs/api.ts runtime template. Plugs into @sun-asterisk/sungen via the capability SPI.",
   "main": "src/index.ts",
   "types": "src/index.ts",
@@ -13,7 +13,7 @@
   "author": "eqe team (engineer & quality) — Sun Asterisk",
   "license": "MIT",
   "dependencies": {
-    "@sun-asterisk/sungen": "3.1.2-beta.104",
+    "@sun-asterisk/sungen": "3.1.2-beta.106",
     "commander": "^14.0.2",
     "yaml": "^2.8.2"
   },

package/src/cli-import.ts

@@ -2,58 +2,183 @@ import { Command } from 'commander';
 import * as fs from 'node:fs';
 import * as path from 'node:path';
 import { parse as parseYaml, stringify as stringifyYaml } from 'yaml';
-import { importOpenApi } from './openapi-import';
+import { importOpenApi, type ImportedApiEntry, type ImportResult } from './openapi-import';
+import { importCsv } from './csv-import';
 
 /**
- * `sungen api …` — API Driver authoring commands (P3, Mode A).
- * For now: `import` an OpenAPI/Swagger doc into a reviewed `api/apis.yaml` catalog.
+ * `sungen api …` — API Driver authoring (Mode A).
+ *   init    — scaffold the API project (kind: api datasource + qa/api + .env key)
+ *   add     — scaffold one area (resource/tag): qa/api/<area>/…
+ *   import  — OpenAPI/Swagger OR CSV/Google-Sheet → apis.yaml, grouped into areas by tag
  */
+
+const CATALOG_HEADER =
+  '# Named-endpoint catalog (API Driver) — generated by `sungen api import`.\n' +
+  '# Review the request shape, wire each `datasource` in datasources.yaml (kind: api),\n' +
+  '# then reference an endpoint from a scenario with `@api:<name>`.\n';
+
+const slug = (s: string) => s.replace(/[^A-Za-z0-9]+/g, '_').replace(/^_+|_+$/g, '').toLowerCase();
+const rel = (p: string) => path.relative(process.cwd(), p);
+
+function writeCatalog(file: string, entries: Record<string, ImportedApiEntry>, merge: boolean): void {
+  let out = entries;
+  if (fs.existsSync(file)) {
+    if (!merge) throw new Error(`${rel(file)} already exists — pass --merge to combine.`);
+    const existing = parseYaml(fs.readFileSync(file, 'utf8')) || {};
+    out = { ...existing, ...entries }; // imported entries win on name clash
+  }
+  fs.mkdirSync(path.dirname(file), { recursive: true });
+  fs.writeFileSync(file, CATALOG_HEADER + stringifyYaml(out));
+}
+
+/** Read a source path or http(s) URL (a published Google-Sheet CSV) → text. */
+async function readSource(source: string): Promise<string> {
+  if (/^https?:\/\//.test(source)) {
+    const res = await fetch(source);
+    if (!res.ok) throw new Error(`fetch ${source} → ${res.status}`);
+    return await res.text();
+  }
+  if (!fs.existsSync(source)) throw new Error(`source not found: ${source}`);
+  return fs.readFileSync(source, 'utf8');
+}
+
+/** Detect OpenAPI vs CSV: a parseable doc with `paths`/`openapi`/`swagger` is OpenAPI; else CSV. */
+function importAny(source: string, text: string, datasource: string): { result: ImportResult; kind: 'openapi' | 'csv' } {
+  if (/\.csv$/i.test(source)) return { result: importCsv(text, datasource), kind: 'csv' };
+  let doc: any;
+  try { doc = parseYaml(text); } catch { doc = undefined; } // yaml lib also parses JSON
+  if (doc && typeof doc === 'object' && (doc.paths || doc.openapi || doc.swagger)) {
+    return { result: importOpenApi(doc, datasource), kind: 'openapi' };
+  }
+  return { result: importCsv(text, datasource), kind: 'csv' };
+}
+
 export function registerApiCommand(program: Command): void {
-  const api = program.command('api').description('API Driver authoring (import an OpenAPI spec into an apis.yaml catalog)');
+  const api = program.command('api').description('API Driver authoring — init · add · import');
 
+  // ── init ─────────────────────────────────────────────────────────────────
   api
-    .command('import <openapi>')
-    .description('Generate api/apis.yaml entries from an OpenAPI/Swagger document')
-    .option('--screen <name>', 'Write to qa/screens/<name>/api/apis.yaml (else shared qa/api/apis.yaml)')
-    .option('--datasource <name>', 'Datasource name to assign to imported endpoints', 'app_api')
-    .option('--merge', 'Merge into an existing apis.yaml instead of refusing to overwrite')
-    .action((openapi: string, opts: { screen?: string; datasource: string; merge?: boolean }) => {
+    .command('init')
+    .description('Scaffold an API project: a kind:api datasource + qa/api/ + the .env.qa URL key')
+    .requiredOption('--base-url <url>', 'API base URL (stored in .env.qa, referenced as ${<DS>_URL})')
+    .option('--datasource <name>', 'Datasource name', 'app_api')
+    .action((opts: { baseUrl: string; datasource: string }) => {
       try {
-        if (!fs.existsSync(openapi)) throw new Error(`OpenAPI file not found: ${openapi}`);
-        const text = fs.readFileSync(openapi, 'utf8');
-        const doc = parseYaml(text); // YAML parser also accepts JSON
-        const { entries, count, skipped } = importOpenApi(doc, opts.datasource);
-        if (count === 0) throw new Error(`No operations imported (${skipped.join('; ') || 'empty paths'}).`);
-
-        const outDir = opts.screen
-          ? path.join(process.cwd(), 'qa', 'screens', opts.screen, 'api')
-          : path.join(process.cwd(), 'qa', 'api');
-        const outFile = path.join(outDir, 'apis.yaml');
-
-        let merged = entries;
-        if (fs.existsSync(outFile)) {
-          if (!opts.merge) throw new Error(`${path.relative(process.cwd(), outFile)} already exists — pass --merge to combine.`);
-          const existing = parseYaml(fs.readFileSync(outFile, 'utf8')) || {};
-          merged = { ...existing, ...entries }; // imported entries win on name clash
+        const ds = opts.datasource;
+        const envKey = `${ds.toUpperCase()}_URL`;
+        fs.mkdirSync(path.join(process.cwd(), 'qa', 'api'), { recursive: true });
+
+        // datasources.yaml — merge (never clobber other datasources)
+        const dsFile =
+          [path.join(process.cwd(), 'qa', 'datasources.yaml'), path.join(process.cwd(), 'datasources.yaml')].find((f) => fs.existsSync(f)) ||
+          path.join(process.cwd(), 'qa', 'datasources.yaml');
+        const dsDoc: any = fs.existsSync(dsFile) ? parseYaml(fs.readFileSync(dsFile, 'utf8')) || {} : {};
+        dsDoc.datasources = dsDoc.datasources || {};
+        if (!dsDoc.datasources[ds]) {
+          dsDoc.datasources[ds] = { kind: 'api', base_url: `\${${envKey}}`, timeout_ms: 15000 };
+          fs.mkdirSync(path.dirname(dsFile), { recursive: true });
+          fs.writeFileSync(dsFile, stringifyYaml(dsDoc));
+          console.log(`✓ datasource "${ds}" (kind: api) → ${rel(dsFile)}`);
+        } else {
+          console.log(`• datasource "${ds}" already in ${rel(dsFile)} — left as is`);
         }
 
-        fs.mkdirSync(outDir, { recursive: true });
-        const header = '# Named-endpoint catalog (API Driver) — generated from an OpenAPI spec by `sungen api import`.\n' +
-          '# Review the SQL-free request shape, wire each `datasource` in datasources.yaml (kind: api),\n' +
-          '# then reference an endpoint from a scenario with `@api:<name>`.\n';
-        fs.writeFileSync(outFile, header + stringifyYaml(merged));
+        // .env.qa (real value, gitignored) + .env.qa.example (key only)
+        upsertEnv(path.join(process.cwd(), '.env.qa'), envKey, opts.baseUrl);
+        upsertEnv(path.join(process.cwd(), '.env.qa.example'), envKey, '');
+        console.log(`✓ ${envKey} → .env.qa (value) + .env.qa.example (key)`);
+        console.log('\nNext: `sungen api import <openapi|csv|sheet-url>` or `sungen api add --area <name>`.\n');
+      } catch (e) { fail(e); }
+    });
 
-        console.log(`\n✓ Imported ${count} endpoint(s) → ${path.relative(process.cwd(), outFile)}`);
-        for (const name of Object.keys(entries).slice(0, 12)) {
-          const e = entries[name];
-          console.log(`   ${name}: ${e.method} ${e.path}${e.expect ? ` → ${e.expect.status}` : ''}`);
+  // ── add ──────────────────────────────────────────────────────────────────
+  api
+    .command('add')
+    .description('Scaffold an API area (resource/tag): qa/api/<area>/{features,test-data,requirements,api}')
+    .requiredOption('--area <name>', 'Area / resource name (e.g. orders, users, auth)')
+    .action((opts: { area: string }) => {
+      try {
+        const area = slug(opts.area);
+        if (!area) throw new Error(`invalid --area "${opts.area}"`);
+        const base = path.join(process.cwd(), 'qa', 'api', area);
+        if (fs.existsSync(base)) throw new Error(`${rel(base)} already exists.`);
+        for (const d of ['features', 'test-data', 'requirements', 'api']) fs.mkdirSync(path.join(base, d), { recursive: true });
+        fs.writeFileSync(path.join(base, 'features', `${area}.feature`), featureStub(area));
+        fs.writeFileSync(path.join(base, 'test-data', `${area}.yaml`), `# Test data for the ${area} API area.\n`);
+        fs.writeFileSync(path.join(base, 'requirements', 'spec.md'), `# ${area} API\n\n_Describe the ${area} endpoints, rules, and expected responses here (or import an OpenAPI/CSV)._\n`);
+        fs.writeFileSync(path.join(base, 'api', 'apis.yaml'), CATALOG_HEADER);
+        console.log(`✓ area "${area}" → ${rel(base)}/`);
+        console.log(`\nNext: import or hand-author ${rel(path.join(base, 'api', 'apis.yaml'))}, then add @api scenarios in features/${area}.feature.\n`);
+      } catch (e) { fail(e); }
+    });
+
+  // ── import ───────────────────────────────────────────────────────────────
+  api
+    .command('import <source>')
+    .description('Generate apis.yaml from an OpenAPI/Swagger doc OR a CSV / published Google-Sheet URL')
+    .option('--datasource <name>', 'Default datasource for imported endpoints', 'app_api')
+    .option('--single', 'Write one shared qa/api/apis.yaml (default: one file per area)')
+    .option('--screen <name>', 'Legacy: write to qa/screens/<name>/api/apis.yaml')
+    .option('--merge', 'Merge into an existing catalog instead of refusing to overwrite')
+    .action(async (source: string, opts: { datasource: string; single?: boolean; screen?: string; merge?: boolean }) => {
+      try {
+        const text = await readSource(source);
+        const { result, kind } = importAny(source, text, opts.datasource);
+        const { entries, count, skipped } = result;
+        if (count === 0) throw new Error(`No endpoints imported (${skipped.join('; ') || 'empty source'}).`);
+
+        const written: string[] = [];
+        if (opts.screen) {
+          const f = path.join(process.cwd(), 'qa', 'screens', opts.screen, 'api', 'apis.yaml');
+          writeCatalog(f, entries, !!opts.merge);
+          written.push(rel(f));
+        } else if (opts.single) {
+          const f = path.join(process.cwd(), 'qa', 'api', 'apis.yaml');
+          writeCatalog(f, entries, !!opts.merge);
+          written.push(rel(f));
+        } else {
+          // group by area → qa/api/<area>/api/apis.yaml
+          const byArea: Record<string, Record<string, ImportedApiEntry>> = {};
+          for (const [name, e] of Object.entries(entries)) (byArea[e.area || 'root'] ||= {})[name] = e;
+          for (const [area, group] of Object.entries(byArea)) {
+            const f = path.join(process.cwd(), 'qa', 'api', area, 'api', 'apis.yaml');
+            writeCatalog(f, group, !!opts.merge);
+            written.push(`${rel(f)} (${Object.keys(group).length})`);
+          }
         }
-        if (Object.keys(entries).length > 12) console.log(`   … and ${Object.keys(entries).length - 12} more`);
-        if (skipped.length) console.log(`\n⚠ Skipped: ${skipped.join('; ')}`);
-        console.log('\nNext: set the `datasource` base_url + auth in datasources.yaml, then `@api:<name>` in a scenario.\n');
-      } catch (error) {
-        console.error('Error:', error instanceof Error ? error.message : error);
-        process.exit(1);
-      }
+
+        console.log(`\n✓ Imported ${count} endpoint(s) from ${kind} source:`);
+        for (const w of written) console.log(`   → ${w}`);
+        if (skipped.length) console.log(`\n⚠ Skipped: ${skipped.slice(0, 10).join('; ')}${skipped.length > 10 ? ` … (+${skipped.length - 10})` : ''}`);
+        console.log('\nNext: wire the `datasource` base_url + auth in datasources.yaml, then `@api:<name>` in a scenario.\n');
+      } catch (e) { fail(e); }
     });
 }
+
+function upsertEnv(file: string, key: string, value: string): void {
+  const line = `${key}=${value}`;
+  if (!fs.existsSync(file)) { fs.writeFileSync(file, line + '\n'); return; }
+  const content = fs.readFileSync(file, 'utf8');
+  if (new RegExp(`^\\s*${key}\\s*=`, 'm').test(content)) return; // already present — don't overwrite a real value
+  fs.writeFileSync(file, content.replace(/\n?$/, '\n') + line + '\n');
+}
+
+function featureStub(area: string): string {
+  return [
+    `@area:${area}`,
+    `Feature: ${area} API`,
+    '',
+    '  # Reference an endpoint from api/apis.yaml with @api:<name>; assert the bound response with',
+    '  # expect {{<name>.status}} / {{<name>.body.<path>}}. Example:',
+    '  #',
+    `  # @api:get_${area}`,
+    `  # Scenario: VP-API-001 Fetching ${area} returns 200`,
+    `  #   Then expect {{get_${area}.status}} is 200`,
+    '',
+  ].join('\n');
+}
+
+function fail(e: unknown): never {
+  console.error('Error:', e instanceof Error ? e.message : e);
+  process.exit(1);
+}

package/src/csv-import.ts

@@ -0,0 +1,101 @@
+/**
+ * CSV / Google-Sheet → apis.yaml importer (API Driver, AP-2).
+ *
+ * Many teams keep their endpoint list in a spreadsheet (Google Sheet → "Publish to web" CSV, or an
+ * exported .csv). This turns that flat table into the same `ImportedApiEntry` catalog shape as the
+ * OpenAPI importer. Deterministic — no AI, no network (the CLI fetches a published-CSV URL; this
+ * module only parses text).
+ *
+ * Columns (header row, case-insensitive; aliases accepted):
+ *   name | area|tag | method | path | params | body | datasource|auth | expect_status | description
+ *   - params: separated by `;` or `|` (commas live inside quoted JSON, so don't use `,` here)
+ *   - body:   a JSON object string, values may be `:param` refs, e.g. {"item_id":":item_id"}
+ *   - expect_status: a number (e.g. 201)
+ */
+import type { ImportedApiEntry, ImportResult } from './openapi-import';
+
+/** RFC-4180-ish parser: handles quoted fields, embedded commas/newlines, and "" escapes. */
+export function parseCsv(text: string): string[][] {
+  const s = text.replace(/\r\n?/g, '\n');
+  const rows: string[][] = [];
+  let field = '';
+  let row: string[] = [];
+  let inQuotes = false;
+  for (let i = 0; i < s.length; i++) {
+    const c = s[i];
+    if (inQuotes) {
+      if (c === '"') {
+        if (s[i + 1] === '"') { field += '"'; i++; } else inQuotes = false;
+      } else field += c;
+    } else if (c === '"') inQuotes = true;
+    else if (c === ',') { row.push(field); field = ''; }
+    else if (c === '\n') { row.push(field); rows.push(row); row = []; field = ''; }
+    else field += c;
+  }
+  if (field.length || row.length) { row.push(field); rows.push(row); }
+  return rows.filter((r) => r.some((cell) => cell.trim() !== '')); // drop blank lines
+}
+
+const ALIASES: Record<string, string> = {
+  tag: 'area', auth: 'datasource', status: 'expect_status', expectstatus: 'expect_status',
+  endpoint: 'path', url: 'path', verb: 'method', key: 'name', desc: 'description',
+};
+const norm = (h: string) => {
+  const k = h.trim().toLowerCase().replace(/[\s-]+/g, '_');
+  return ALIASES[k.replace(/_/g, '')] || ALIASES[k] || k;
+};
+const slug = (s: string) => s.replace(/[^A-Za-z0-9]+/g, '_').replace(/^_+|_+$/g, '').toLowerCase();
+
+/**
+ * Transform a CSV/Sheet table into catalog entries.
+ * @param text   raw CSV
+ * @param datasource default datasource when a row omits one (default 'app_api')
+ */
+export function importCsv(text: string, datasource = 'app_api'): ImportResult {
+  const rows = parseCsv(text);
+  const entries: Record<string, ImportedApiEntry> = {};
+  const skipped: string[] = [];
+  if (rows.length < 2) return { entries, count: 0, skipped: ['CSV has no data rows (need a header + ≥1 row)'] };
+
+  const header = rows[0].map(norm);
+  const col = (r: string[], name: string): string => {
+    const i = header.indexOf(name);
+    return i >= 0 && i < r.length ? r[i].trim() : '';
+  };
+
+  for (let n = 1; n < rows.length; n++) {
+    const r = rows[n];
+    const name = col(r, 'name');
+    const method = col(r, 'method').toUpperCase();
+    const path = col(r, 'path');
+    if (!name || !method || !path) { skipped.push(`row ${n + 1} (missing name/method/path)`); continue; }
+    if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(name)) { skipped.push(`row ${n + 1} ("${name}" is not a valid catalog key)`); continue; }
+    if (entries[name]) { skipped.push(`row ${n + 1} (duplicate name "${name}")`); continue; }
+
+    const area = slug(col(r, 'area')) || slug(path.replace(/^\/+/, '').split('/')[0]) || 'root';
+    const pathParams = (path.match(/:([A-Za-z_][A-Za-z0-9_]*)/g) || []).map((x) => x.slice(1));
+    const declared = col(r, 'params').split(/[;|]/).map((p) => p.trim()).filter(Boolean);
+    const params = [...new Set([...pathParams, ...declared])];
+
+    let body: Record<string, string> | undefined;
+    const bodyCell = col(r, 'body');
+    if (bodyCell) {
+      try { body = JSON.parse(bodyCell); } catch { skipped.push(`row ${n + 1} (body is not valid JSON — kept no body)`); }
+    }
+    const statusCell = col(r, 'expect_status');
+    const status = /^\d+$/.test(statusCell) ? Number(statusCell) : undefined;
+    const description = col(r, 'description') || undefined;
+
+    entries[name] = {
+      datasource: col(r, 'datasource') || datasource,
+      area,
+      ...(description ? { description } : {}),
+      method,
+      path,
+      ...(params.length ? { params } : {}),
+      ...(body ? { body } : {}),
+      ...(status != null ? { expect: { status } } : {}),
+    };
+  }
+  return { entries, count: Object.keys(entries).length, skipped };
+}

package/src/index.ts

@@ -19,6 +19,7 @@ import { registerApiCommand } from './cli-import';
 
 export { importOpenApi } from './openapi-import';
 export type { ImportedApiEntry, ImportResult } from './openapi-import';
+export { importCsv, parseCsv } from './csv-import';
 
 /**
  * API precondition codegen: `@api:<name>[(p={{v}},…)]` → run the catalog request and bind the

package/src/openapi-import.ts

@@ -8,6 +8,7 @@
  */
 export interface ImportedApiEntry {
   datasource: string;
+  area: string;            // resource/tag group (OpenAPI tag, else first path segment)
   description?: string;
   method: string;
   path: string;
@@ -18,6 +19,20 @@ export interface ImportedApiEntry {
 
 const METHODS = ['get', 'post', 'put', 'patch', 'delete', 'head', 'options'];
 
+/** A safe lower-snake slug for area/tag names. */
+function slug(s: string): string {
+  return s.replace(/[^A-Za-z0-9]+/g, '_').replace(/^_+|_+$/g, '').toLowerCase();
+}
+
+/** Group an operation by its first OpenAPI tag (à la GitHub/Google/Swagger); fall back to the first
+ *  path segment (`/orders/{id}` → `orders`). Deterministic — no AI. */
+function areaOf(op: any, rawPath: string): string {
+  const tag = Array.isArray(op?.tags) && typeof op.tags[0] === 'string' ? slug(op.tags[0]) : '';
+  if (tag) return tag;
+  const seg = rawPath.replace(/^\/+/, '').split('/')[0] || '';
+  return slug(seg) || 'root';
+}
+
 /** A safe catalog name from an OpenAPI path + method (fallback when there's no operationId). */
 function fallbackName(method: string, path: string): string {
   const slug = path.replace(/[{}]/g, '').replace(/[^A-Za-z0-9]+/g, '_').replace(/^_+|_+$/g, '').toLowerCase() || 'root';
@@ -86,6 +101,7 @@ export function importOpenApi(doc: any, datasource = 'app_api'): ImportResult {
       const status = expectedStatus(op.responses);
       entries[name] = {
         datasource,
+        area: areaOf(op, rawPath),
         description: typeof op.summary === 'string' ? op.summary : undefined,
         method: method.toUpperCase(),
         path: catalogPath,