@aifabrix/builder 2.49.1 → 2.50.0

package/packages/builder-api/README.md

@@ -0,0 +1,120 @@
+# @aifabrix/builder-api (154.0)
+
+HTTP orchestration layer for external-system workspace lifecycle. V1 uses the shared
+`lib/programmatic/` adapter over existing Builder CLI libraries (no duplicate facades).
+
+## Run with Builder CLI (recommended)
+
+From the **aifabrix-builder** repository root:
+
+```bash
+# From aifabrix-builder repo root
+aifabrix auth status
+aifabrix up-platform              # full stack (includes builder-api)
+# or incremental when miso + dataplane already run:
+aifabrix up-builder-api
+aifabrix resolve builder-api      # refresh .env from env.template
+aifabrix run builder-api --reload
+```
+
+Hot reload (bind-mount repo root, same pattern as dataplane / miso-controller):
+
+```bash
+pnpm run dev-reload
+# or: aifabrix run builder-api --reload
+```
+
+Logs, stop, and lifecycle match other Builder-managed apps:
+
+```bash
+aifabrix logs builder-api
+aifabrix down-app builder-api
+```
+
+Health (adjust port for your `developer-id`): `GET http://localhost:3105/api/v1/builder/health` (dev 1 → `3005 + 100`).
+
+ReDoc (same as dataplane / miso-controller): `GET http://localhost:3105/api/docs`  
+OpenAPI JSON: `GET /api/v1/builder/api-docs/openapi.json`
+
+Platform integration SSOT lives under `integration/` in the builder repo. Publish Enterprise MCP anchor and repository provider integrations before workspace flows:
+
+```bash
+aifabrix upload enterprise-integration-lifecycle --probe
+aifabrix upload <repository-provider-systemKey> --probe
+pnpm run dev-reload   # builder-api container
+```
+
+See `integration/README.md` in the builder repo.
+
+## Run on host (development / tests only)
+
+For Jest e2e and quick iteration without Docker, resolve from `builder/builder-api/env.template` (uses `url://` tokens like miso-controller/dataplane) or set localhost URLs manually:
+
+```bash
+aifabrix resolve builder-api   # preferred — expands url://miso-controller-internal, url://dataplane-internal, url://keycloak-*
+
+# or manual host overrides:
+export WORKSPACE_ROOT=/tmp/builder-workspaces
+export MISO_CLIENTID=miso-controller-dev-builder-api
+export MISO_CLIENTSECRET=<from aifabrix app register builder-api>
+export MISO_CONTROLLER_URL=http://localhost:3100/miso
+export DATAPLANE_URL=http://localhost:3101/data
+export DEFAULT_ENVIRONMENT=dev
+
+pnpm run dev
+# or Nest bootstrap: npm run builder-api:nest
+```
+
+## Workspace cleanup
+
+Scheduled purge of stale draft workspaces (env `WORKSPACE_DRAFT_TTL_DAYS`, `WORKSPACE_INACTIVE_MAX_DAYS`):
+
+```bash
+npm run builder-api:cleanup
+npm run builder-api:cleanup -- --dry-run
+```
+
+In Docker, schedule via platform cron or sidecar invoking `node bin/builder-api-cleanup.js`.
+
+## Package layout
+
+```text
+builder/builder-api/                    # SSOT (edit here; npm run sync-json → template)
+templates/applications/builder-api/   # Shipped template copy
+packages/builder-api/
+  src/main.ts                           # Nest bootstrap → nest-http-bridge
+  src/app.module.ts
+  MANUAL-CHECKLIST.md
+lib/programmatic/                       # Shared HTTP handlers + Enterprise MCP client
+```
+
+Compile TypeScript: `npm run builder-api:build` (output in `packages/builder-api/dist/`).
+
+## 154.1 routes (Enterprise MCP client)
+
+Provider-neutral HTTP orchestration — remote repository/template via Dataplane Enterprise MCP:
+
+- `POST /systems/{systemKey}/template/materialize`
+- `GET /systems/{systemKey}/workspace/status`
+- `POST /systems/{systemKey}/workspace/diff` / `workspace/reset`
+- `POST /systems/{systemKey}/repository/sync` / `create-review` / `dispatch-pipeline`
+- `GET /systems/{systemKey}/repository/status`
+
+Governance gates use existing Dataplane lifecycle / verify-trust APIs (no can-execute proxy in builder-api V1).
+
+Architecture CI: `npm run check:builder-api-arch`
+
+All other routes require `Authorization: Bearer <token from aifabrix login>`.
+
+See `MANUAL-CHECKLIST.md` for manual verification, `docs/external-systems.md` and `docs/wizard.md` for integration workflow context.
+
+## Nest migration
+
+This package currently reuses the Node HTTP server in `lib/programmatic/builder-api-server.js`.
+NestJS scaffold wraps the same `route-handlers` module via `nest-http-bridge.js`.
+
+## Not builder-server
+
+This service is **not** the dev onboarding `builder-server`. It is stateless multi-user workspace
+orchestration with Dataplane as governance authority. Do **not** use ad hoc `docker compose` under
+`packages/builder-api/` — use `aifabrix run builder-api` for the full application lifecycle.

package/packages/builder-api/package.json

@@ -0,0 +1,30 @@
+{
+  "name": "@aifabrix/builder-api",
+  "version": "0.1.0",
+  "description": "AI Fabrix Builder API — HTTP orchestration for external system lifecycle (plan 154.0)",
+  "private": true,
+  "main": "../../lib/programmatic/builder-api-server.js",
+  "bin": {
+    "builder-api": "../../bin/builder-api-spike.js",
+    "builder-api-nest": "../../bin/builder-api-nest.js"
+  },
+  "scripts": {
+    "build": "tsc -p tsconfig.json",
+    "start": "node ../../bin/builder-api-spike.js",
+    "start:nest": "node ../../bin/builder-api-nest.js",
+    "cleanup": "node ../../bin/builder-api-cleanup.js"
+  },
+  "engines": {
+    "node": ">=18.0.0"
+  },
+  "dependencies": {
+    "@aifabrix/miso-client": "4.19.0"
+  },
+  "peerDependencies": {
+    "@nestjs/common": "^10.0.0",
+    "@nestjs/core": "^10.0.0",
+    "@nestjs/platform-express": "^10.0.0",
+    "reflect-metadata": "^0.2.0",
+    "rxjs": "^7.8.0"
+  }
+}

package/packages/builder-api/src/app.module.ts

@@ -0,0 +1,7 @@
+import { Module } from '@nestjs/common';
+
+/**
+ * Root Nest module — HTTP routing is delegated to lib/programmatic via nest-http-bridge.
+ */
+@Module({})
+export class AppModule {}

package/packages/builder-api/src/auth/auth.module.ts

@@ -0,0 +1,5 @@
+import { Module } from '@nestjs/common';
+
+/** Placeholder for future BearerAuthGuard wiring (V1 auth runs in lib/programmatic). */
+@Module({})
+export class AuthModule {}

package/packages/builder-api/src/external-system/external-system.module.ts

@@ -0,0 +1,5 @@
+import { Module } from '@nestjs/common';
+
+/** Placeholder for route-group modules; handlers live in lib/programmatic today. */
+@Module({})
+export class ExternalSystemModule {}

package/packages/builder-api/src/main.ts

@@ -0,0 +1,16 @@
+import 'reflect-metadata';
+import { NestFactory } from '@nestjs/core';
+import { AppModule } from './app.module';
+
+async function bootstrap() {
+  const app = await NestFactory.create(AppModule, { bodyParser: false });
+  // eslint-disable-next-line @typescript-eslint/no-require-imports
+  const { wireBuilderApiHttpBridge, BASE } = require('../../../lib/programmatic/nest-http-bridge');
+  wireBuilderApiHttpBridge(app.getHttpAdapter().getInstance());
+  const port = Number(process.env.PORT || 3005);
+  await app.listen(port);
+  // eslint-disable-next-line no-console
+  console.log(`builder-api (Nest) listening on http://localhost:${port}${BASE}/health`);
+}
+
+bootstrap();

package/packages/builder-api/tsconfig.json

@@ -0,0 +1,15 @@
+{
+  "compilerOptions": {
+    "module": "commonjs",
+    "target": "ES2020",
+    "outDir": "dist",
+    "rootDir": "src",
+    "strict": true,
+    "esModuleInterop": true,
+    "experimentalDecorators": true,
+    "emitDecoratorMetadata": true,
+    "skipLibCheck": true,
+    "declaration": false
+  },
+  "include": ["src/**/*.ts"]
+}

package/scripts/check-builder-api-architecture-grep.js

@@ -0,0 +1,79 @@
+#!/usr/bin/env node
+/**
+ * Architecture grep gate for builder-api (plan 154.1).
+ *
+ * Fails when provider-specific strings appear under lib/programmatic or packages/builder-api.
+ */
+
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+
+const ROOT = path.join(__dirname, '..');
+const SCAN_DIRS = ['lib/programmatic', 'packages/builder-api'];
+const FORBIDDEN = /\b(github|gitlab|bitbucket|octokit|azuredevops|azure devops|git clone)\b/i;
+const ALLOWLIST = new Set([
+  path.normalize('lib/programmatic/openapi-descriptions.js'),
+  path.normalize('lib/programmatic/openapi-spec.js'),
+  path.normalize('packages/builder-api/MANUAL-CHECKLIST.md')
+]);
+
+/**
+ * Recursively collect scannable files under a directory.
+ * @param {string} dir - Directory path to scan
+ * @param {string[]} files - Output file list
+ * @returns {void}
+ */
+function walk(dir, files) {
+  if (!fs.existsSync(dir)) {
+    return;
+  }
+  for (const name of fs.readdirSync(dir)) {
+    const full = path.join(dir, name);
+    const stat = fs.statSync(full);
+    if (stat.isDirectory()) {
+      walk(full, files);
+      continue;
+    }
+    if (!/\.(js|ts|md|json|yml|yaml)$/.test(name)) {
+      continue;
+    }
+    files.push(full);
+  }
+}
+
+/** @type {string[]} */
+const files = [];
+for (const rel of SCAN_DIRS) {
+  walk(path.join(ROOT, rel), files);
+}
+
+/** @type {Array<{ file: string, line: number, text: string }>} */
+const violations = [];
+
+for (const file of files) {
+  const rel = path.normalize(path.relative(ROOT, file));
+  if (ALLOWLIST.has(rel)) {
+    continue;
+  }
+  const content = fs.readFileSync(file, 'utf8');
+  content.split('\n').forEach((line, index) => {
+    if (FORBIDDEN.test(line)) {
+      violations.push({ file: rel, line: index + 1, text: line.trim() });
+    }
+  });
+}
+
+if (violations.length > 0) {
+  // eslint-disable-next-line no-console
+  console.error('builder-api architecture grep failed:');
+  for (const row of violations) {
+    // eslint-disable-next-line no-console
+    console.error(`  ${row.file}:${row.line}: ${row.text}`);
+  }
+  process.exit(1);
+}
+
+// eslint-disable-next-line no-console
+console.log('builder-api architecture grep passed');

package/scripts/docker_image_content_gate.js

@@ -0,0 +1,47 @@
+#!/usr/bin/env node
+/**
+ * Fail when a built Docker image contains dev-only paths (plan 154.3).
+ *
+ * @fileoverview builder-api image content gate
+ * @author AI Fabrix Team
+ * @version 1.0.0
+ */
+
+'use strict';
+
+const { checkImage } = require('./lib/docker-image-content-gate-core');
+
+/**
+ * @param {string[]} argv
+ * @returns {number}
+ */
+function main(argv) {
+  const args = argv.slice(2);
+  if (args.length < 2) {
+    console.error('Usage: docker_image_content_gate.js <image> <profile>');
+    return 1;
+  }
+
+  const [image, profile] = args;
+  try {
+    const violations = checkImage(image, profile);
+    if (violations.length > 0) {
+      console.error(`Image ${image} failed ${profile} content gate:`);
+      for (const path of violations) {
+        console.error(`  - forbidden: ${path}`);
+      }
+      return 1;
+    }
+    console.log(`OK: ${image} passed ${profile} content gate`);
+    return 0;
+  } catch (error) {
+    console.error(`ERROR: ${error.message}`);
+    return 1;
+  }
+}
+
+if (require.main === module) {
+  process.exit(main(process.argv));
+}
+
+module.exports = { main };

package/scripts/docker_image_content_gate.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env sh
+# Plan 154.3 — verify a built builder-api image does not contain dev-only trees.
+set -eu
+ROOT="$(CDPATH= cd -- "$(dirname "$0")/.." && pwd)"
+exec node "$ROOT/scripts/docker_image_content_gate.js" "$@"

package/scripts/export-builder-api-evidence.js

@@ -0,0 +1,262 @@
+#!/usr/bin/env node
+/**
+ * Export builder-api help/knowledge fixtures to Evidence Fabrix v2 bulk-load JSON.
+ *
+ * @fileoverview One-time / repeatable generator for Role Assistant evidence import
+ * @author AI Fabrix Team
+ * @version 1.0.0
+ */
+
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+
+const REPO_ROOT = path.join(__dirname, '..');
+const DEFAULT_OUT = path.join(
+  REPO_ROOT,
+  '..',
+  'aifabrix-dataplane',
+  'integration',
+  'aifabrix-builder-api',
+  'aifabrix-builder-api-evidence.json'
+);
+
+const EVIDENCE_PATTERNS = path.join(REPO_ROOT, 'docs', 'builder-help', 'evidence-patterns.json');
+const GOLDEN_DIR = path.join(REPO_ROOT, 'docs', 'builder-help', 'golden-examples');
+const PLATFORM_ROLES = path.join(REPO_ROOT, 'docs', 'builder-help', 'platform-roles.json');
+const RESOURCE_CATALOG = path.join(REPO_ROOT, 'docs', 'builder-help', 'resource-type-catalog.json');
+const HELP_MD_DIR = path.join(REPO_ROOT, 'lib', 'programmatic', 'help-content');
+
+/** @type {Array<{ file: string, resourceType: string, order: number }>} */
+const MD_TOPICS = [
+  { file: 'overview.md', resourceType: 'workspace', order: 10 },
+  { file: 'workflow.md', resourceType: 'workspace', order: 20 },
+  { file: 'subscription-guide.md', resourceType: 'system', order: 30 },
+  { file: 'cip-overview.md', resourceType: 'task', order: 40 },
+  { file: 'dimensions-guide.md', resourceType: 'policy', order: 50 },
+  { file: 'channel-onboarding.md', resourceType: 'channel', order: 60 },
+  { file: 'connected-system-ui.md', resourceType: 'workspace', order: 70 }
+];
+
+/**
+ * @param {string} text
+ * @returns {string}
+ */
+function stripMarkdown(text) {
+  return String(text || '')
+    .replace(/\[([^\]]+)\]\([^)]+\)/g, '$1')
+    .replace(/https?:\/\/[^\s)]+/gi, '')
+    .replace(/^#+\s*/gm, '')
+    .replace(/\s{2,}/g, ' ')
+    .trim();
+}
+
+/**
+ * @param {string} fileName
+ * @returns {{ title: string, summary: string, body: string }}
+ */
+function parseMarkdownTopic(fileName) {
+  const raw = fs.readFileSync(path.join(HELP_MD_DIR, fileName), 'utf8');
+  const lines = raw.split(/\r?\n/);
+  let title = path.basename(fileName, '.md');
+  const bodyLines = [];
+  for (const line of lines) {
+    if (line.startsWith('# ')) {
+      title = stripMarkdown(line.slice(2));
+    } else if (line.trim()) {
+      bodyLines.push(line);
+    }
+  }
+  const body = stripMarkdown(bodyLines.join('\n'));
+  const summary = body.slice(0, 240);
+  return { title, summary, body };
+}
+
+/**
+ * @param {Object} row
+ * @returns {Object}
+ */
+function normalizeRow(row) {
+  const out = {
+    resourceType: String(row.resourceType || 'workspace').trim(),
+    title: String(row.title || '').trim(),
+    when: String(row.when || '').trim(),
+    expectedBehavior: String(row.expectedBehavior || '').trim(),
+    order: Number.isFinite(row.order) ? row.order : 100,
+    status: row.status === 'disabled' ? 'disabled' : 'active'
+  };
+  if (row.roleKey) {
+    out.roleKey = String(row.roleKey).trim();
+  }
+  if (row.description) {
+    out.description = String(row.description).trim();
+  }
+  return out;
+}
+
+/**
+ * @returns {Object[]}
+ */
+function rowsFromEvidencePatterns() {
+  const doc = JSON.parse(fs.readFileSync(EVIDENCE_PATTERNS, 'utf8'));
+  return (doc.patterns || []).map((p, i) =>
+    normalizeRow({
+      resourceType: p.resourceType || 'document',
+      title: p.title,
+      description: [p.summary, p.role ? `Audience: ${p.role}` : null, (p.steps || []).join(' → ')].filter(Boolean).join(' '),
+      when: p.summary || p.title,
+      expectedBehavior: (p.steps || []).join(' '),
+      order: 100 + i
+    })
+  );
+}
+
+/**
+ * @returns {Object[]}
+ */
+function rowsFromGoldenExamples() {
+  const rows = [];
+  let order = 200;
+  for (const file of fs.readdirSync(GOLDEN_DIR).sort()) {
+    if (!file.endsWith('.json')) {
+      continue;
+    }
+    const doc = JSON.parse(fs.readFileSync(path.join(GOLDEN_DIR, file), 'utf8'));
+    const traversal = (doc.metadataTraversal || []).join(', ');
+    rows.push(
+      normalizeRow({
+        resourceType: doc.resourceType || 'document',
+        title: `Golden example: ${doc.title || doc.key}`,
+        description: [doc.summary, doc.role ? `Audience: ${doc.role}` : null].filter(Boolean).join(' '),
+        when: `Designing a ${doc.resourceType || 'document'} integration similar to ${doc.key}`,
+        expectedBehavior: [
+          doc.summary,
+          doc.subscriptionGuidance,
+          traversal ? `Traverse sections: ${traversal}` : null,
+          (doc.recommendedSectionKeys || []).length
+            ? `Recommended sections: ${doc.recommendedSectionKeys.join(', ')}`
+            : null
+        ]
+          .filter(Boolean)
+          .join(' '),
+        order: order++
+      })
+    );
+  }
+  return rows;
+}
+
+/**
+ * @returns {Object[]}
+ */
+function rowsFromMarkdownTopics() {
+  return MD_TOPICS.map((t) => {
+    const parsed = parseMarkdownTopic(t.file);
+    return normalizeRow({
+      resourceType: t.resourceType,
+      title: parsed.title,
+      description: parsed.summary,
+      when: `Agent needs builder-api guidance for ${parsed.title}`,
+      expectedBehavior: parsed.body,
+      order: t.order
+    });
+  });
+}
+
+/**
+ * @returns {Object[]}
+ */
+function rowsFromPlatformRoles() {
+  const doc = JSON.parse(fs.readFileSync(PLATFORM_ROLES, 'utf8'));
+  const roleLines = (doc.roles || [])
+    .map((r) => `${r.value}: ${r.description}`)
+    .join(' ');
+  return [
+    normalizeRow({
+      resourceType: 'policy',
+      title: 'Platform RBAC roles for facade integrations',
+      description: 'Dataplane platform roles for orchestration facade systems (not per-system admin/reader).',
+      when: 'Patching system.rolesPermissions or repair rbac on a builder-api facade system',
+      expectedBehavior: `Use rbacPreset platform. Allowed roles: ${roleLines}. Avoid {systemKey}-admin or {systemKey}-reader on facade APIs.`,
+      order: 5
+    })
+  ];
+}
+
+/**
+ * @returns {Object[]}
+ */
+function rowsFromResourceCatalog() {
+  const doc = JSON.parse(fs.readFileSync(RESOURCE_CATALOG, 'utf8'));
+  const byCategory = {};
+  for (const rt of doc.resourceTypes || []) {
+    const cat = rt.category || 'other';
+    if (!byCategory[cat]) {
+      byCategory[cat] = [];
+    }
+    byCategory[cat].push(`${rt.key} (${rt.description})`);
+  }
+  const summary = Object.entries(byCategory)
+    .map(([cat, items]) => `${cat}: ${items.slice(0, 8).join('; ')}`)
+    .join(' ');
+  return [
+    normalizeRow({
+      resourceType: 'document',
+      title: 'Resource type catalog for datasource manifests',
+      description: 'Pick catalog keys that match business semantics; challenge generic placeholders.',
+      when: 'Choosing datasource.resourceType for a new or edited datasource',
+      expectedBehavior: `Compare purpose to catalog entries. Minimum system keys: ${(doc.minimumSystemKeys || []).join(', ')}. Categories: ${summary}`,
+      order: 8
+    })
+  ];
+}
+
+/**
+ * @returns {Object[]}
+ */
+function rowsFromManifestGuides() {
+  return [
+    normalizeRow({
+      resourceType: 'policy',
+      title: 'Manifest section keys and dependency order',
+      when: 'Agent edits external system or datasource manifest sections',
+      expectedBehavior:
+        'Read one section at a time via manifest section APIs. After metadata or capability patches, run repair hooks for fieldMappings, expose, and rbac. Use dependency graph: downstream sections derive from upstream patches.',
+      order: 15
+    }),
+    normalizeRow({
+      resourceType: 'workspace',
+      title: 'Repair hooks and CLI flag mapping',
+      when: 'Manifest repair is required after structural edits',
+      expectedBehavior:
+        'POST manifest repair with explicit hooks (fieldMappings, expose, rbac, sync, test). Align with repair CLI flags. Validate workspace before upload or certify.',
+      order: 16
+    }),
+    normalizeRow({
+      resourceType: 'workspace',
+      title: 'Section quality and missing inputs',
+      when: 'GET manifest section returns quality.missingInputs',
+      expectedBehavior:
+        'Collect required missing inputs from the user before PATCH. Do not guess values. Include sectionReadToken from GET in PATCH body.',
+      order: 17
+    })
+  ];
+}
+
+function main() {
+  const outPath = path.resolve(process.argv[2] || DEFAULT_OUT);
+  const rows = [
+    ...rowsFromPlatformRoles(),
+    ...rowsFromResourceCatalog(),
+    ...rowsFromMarkdownTopics(),
+    ...rowsFromManifestGuides(),
+    ...rowsFromEvidencePatterns(),
+    ...rowsFromGoldenExamples()
+  ];
+  fs.mkdirSync(path.dirname(outPath), { recursive: true });
+  fs.writeFileSync(outPath, `${JSON.stringify(rows, null, 2)}\n`, 'utf8');
+  process.stdout.write(`Wrote ${rows.length} evidence row(s) to ${outPath}\n`);
+}
+
+main();

package/scripts/lib/builder-api-docker-staging-copy.js

@@ -0,0 +1,60 @@
+/**
+ * Copy staged builder-api runtime tree to disk.
+ *
+ * @fileoverview Filesystem staging for Docker COPY layers
+ * @author AI Fabrix Team
+ * @version 1.0.0
+ */
+
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+
+/**
+ * @param {string} target
+ * @returns {void}
+ */
+function removeDirIfExists(target) {
+  if (fs.existsSync(target)) {
+    fs.rmSync(target, { recursive: true, force: true });
+  }
+}
+
+/**
+ * @param {string} src
+ * @param {string} dest
+ * @returns {void}
+ */
+function copyFileEnsureDir(src, dest) {
+  fs.mkdirSync(path.dirname(dest), { recursive: true });
+  fs.copyFileSync(src, dest);
+}
+
+/**
+ * @param {string} repoRoot
+ * @param {string} destination
+ * @param {string[]} manifest
+ * @returns {string[]}
+ */
+function stageFiles(repoRoot, destination, manifest) {
+  removeDirIfExists(destination);
+  fs.mkdirSync(destination, { recursive: true });
+
+  const staged = [];
+  for (const rel of manifest) {
+    const src = path.join(repoRoot, rel);
+    const dest = path.join(destination, rel);
+    if (!fs.existsSync(src)) {
+      continue;
+    }
+    copyFileEnsureDir(src, dest);
+    staged.push(rel);
+  }
+  return staged.sort();
+}
+
+module.exports = {
+  stageFiles,
+  removeDirIfExists
+};