@aifabrix/builder 2.49.2 → 2.51.0

package/packages/builder-api/src/external-system/external-system.module.ts

@@ -0,0 +1,5 @@
+import { Module } from '@nestjs/common';
+
+/** Placeholder for route-group modules; handlers live in lib/programmatic today. */
+@Module({})
+export class ExternalSystemModule {}

package/packages/builder-api/src/main.ts

@@ -0,0 +1,16 @@
+import 'reflect-metadata';
+import { NestFactory } from '@nestjs/core';
+import { AppModule } from './app.module';
+
+async function bootstrap() {
+  const app = await NestFactory.create(AppModule, { bodyParser: false });
+  // eslint-disable-next-line @typescript-eslint/no-require-imports
+  const { wireBuilderApiHttpBridge, BASE } = require('../../../lib/programmatic/nest-http-bridge');
+  wireBuilderApiHttpBridge(app.getHttpAdapter().getInstance());
+  const port = Number(process.env.PORT || 3005);
+  await app.listen(port);
+  // eslint-disable-next-line no-console
+  console.log(`builder-api (Nest) listening on http://localhost:${port}${BASE}/health`);
+}
+
+bootstrap();

package/packages/builder-api/tsconfig.json

@@ -0,0 +1,15 @@
+{
+  "compilerOptions": {
+    "module": "commonjs",
+    "target": "ES2020",
+    "outDir": "dist",
+    "rootDir": "src",
+    "strict": true,
+    "esModuleInterop": true,
+    "experimentalDecorators": true,
+    "emitDecoratorMetadata": true,
+    "skipLibCheck": true,
+    "declaration": false
+  },
+  "include": ["src/**/*.ts"]
+}

package/scripts/check-builder-api-architecture-grep.js

@@ -0,0 +1,79 @@
+#!/usr/bin/env node
+/**
+ * Architecture grep gate for builder-api (plan 154.1).
+ *
+ * Fails when provider-specific strings appear under lib/programmatic or packages/builder-api.
+ */
+
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+
+const ROOT = path.join(__dirname, '..');
+const SCAN_DIRS = ['lib/programmatic', 'packages/builder-api'];
+const FORBIDDEN = /\b(github|gitlab|bitbucket|octokit|azuredevops|azure devops|git clone)\b/i;
+const ALLOWLIST = new Set([
+  path.normalize('lib/programmatic/openapi-descriptions.js'),
+  path.normalize('lib/programmatic/openapi-spec.js'),
+  path.normalize('packages/builder-api/MANUAL-CHECKLIST.md')
+]);
+
+/**
+ * Recursively collect scannable files under a directory.
+ * @param {string} dir - Directory path to scan
+ * @param {string[]} files - Output file list
+ * @returns {void}
+ */
+function walk(dir, files) {
+  if (!fs.existsSync(dir)) {
+    return;
+  }
+  for (const name of fs.readdirSync(dir)) {
+    const full = path.join(dir, name);
+    const stat = fs.statSync(full);
+    if (stat.isDirectory()) {
+      walk(full, files);
+      continue;
+    }
+    if (!/\.(js|ts|md|json|yml|yaml)$/.test(name)) {
+      continue;
+    }
+    files.push(full);
+  }
+}
+
+/** @type {string[]} */
+const files = [];
+for (const rel of SCAN_DIRS) {
+  walk(path.join(ROOT, rel), files);
+}
+
+/** @type {Array<{ file: string, line: number, text: string }>} */
+const violations = [];
+
+for (const file of files) {
+  const rel = path.normalize(path.relative(ROOT, file));
+  if (ALLOWLIST.has(rel)) {
+    continue;
+  }
+  const content = fs.readFileSync(file, 'utf8');
+  content.split('\n').forEach((line, index) => {
+    if (FORBIDDEN.test(line)) {
+      violations.push({ file: rel, line: index + 1, text: line.trim() });
+    }
+  });
+}
+
+if (violations.length > 0) {
+  // eslint-disable-next-line no-console
+  console.error('builder-api architecture grep failed:');
+  for (const row of violations) {
+    // eslint-disable-next-line no-console
+    console.error(`  ${row.file}:${row.line}: ${row.text}`);
+  }
+  process.exit(1);
+}
+
+// eslint-disable-next-line no-console
+console.log('builder-api architecture grep passed');

package/scripts/docker_image_content_gate.js

@@ -0,0 +1,47 @@
+#!/usr/bin/env node
+/**
+ * Fail when a built Docker image contains dev-only paths (plan 154.3).
+ *
+ * @fileoverview builder-api image content gate
+ * @author AI Fabrix Team
+ * @version 1.0.0
+ */
+
+'use strict';
+
+const { checkImage } = require('./lib/docker-image-content-gate-core');
+
+/**
+ * @param {string[]} argv
+ * @returns {number}
+ */
+function main(argv) {
+  const args = argv.slice(2);
+  if (args.length < 2) {
+    console.error('Usage: docker_image_content_gate.js <image> <profile>');
+    return 1;
+  }
+
+  const [image, profile] = args;
+  try {
+    const violations = checkImage(image, profile);
+    if (violations.length > 0) {
+      console.error(`Image ${image} failed ${profile} content gate:`);
+      for (const path of violations) {
+        console.error(`  - forbidden: ${path}`);
+      }
+      return 1;
+    }
+    console.log(`OK: ${image} passed ${profile} content gate`);
+    return 0;
+  } catch (error) {
+    console.error(`ERROR: ${error.message}`);
+    return 1;
+  }
+}
+
+if (require.main === module) {
+  process.exit(main(process.argv));
+}
+
+module.exports = { main };

package/scripts/docker_image_content_gate.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env sh
+# Plan 154.3 — verify a built builder-api image does not contain dev-only trees.
+set -eu
+ROOT="$(CDPATH= cd -- "$(dirname "$0")/.." && pwd)"
+exec node "$ROOT/scripts/docker_image_content_gate.js" "$@"

package/scripts/export-builder-api-evidence.js

@@ -0,0 +1,262 @@
+#!/usr/bin/env node
+/**
+ * Export builder-api help/knowledge fixtures to Evidence Fabrix v2 bulk-load JSON.
+ *
+ * @fileoverview One-time / repeatable generator for Role Assistant evidence import
+ * @author AI Fabrix Team
+ * @version 1.0.0
+ */
+
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+
+const REPO_ROOT = path.join(__dirname, '..');
+const DEFAULT_OUT = path.join(
+  REPO_ROOT,
+  '..',
+  'aifabrix-dataplane',
+  'integration',
+  'builder-api',
+  'builder-api-evidence.json'
+);
+
+const EVIDENCE_PATTERNS = path.join(REPO_ROOT, 'docs', 'builder-help', 'evidence-patterns.json');
+const GOLDEN_DIR = path.join(REPO_ROOT, 'docs', 'builder-help', 'golden-examples');
+const PLATFORM_ROLES = path.join(REPO_ROOT, 'docs', 'builder-help', 'platform-roles.json');
+const RESOURCE_CATALOG = path.join(REPO_ROOT, 'docs', 'builder-help', 'resource-type-catalog.json');
+const HELP_MD_DIR = path.join(REPO_ROOT, 'lib', 'programmatic', 'help-content');
+
+/** @type {Array<{ file: string, resourceType: string, order: number }>} */
+const MD_TOPICS = [
+  { file: 'overview.md', resourceType: 'workspace', order: 10 },
+  { file: 'workflow.md', resourceType: 'workspace', order: 20 },
+  { file: 'subscription-guide.md', resourceType: 'system', order: 30 },
+  { file: 'cip-overview.md', resourceType: 'task', order: 40 },
+  { file: 'dimensions-guide.md', resourceType: 'policy', order: 50 },
+  { file: 'channel-onboarding.md', resourceType: 'channel', order: 60 },
+  { file: 'connected-system-ui.md', resourceType: 'workspace', order: 70 }
+];
+
+/**
+ * @param {string} text
+ * @returns {string}
+ */
+function stripMarkdown(text) {
+  return String(text || '')
+    .replace(/\[([^\]]+)\]\([^)]+\)/g, '$1')
+    .replace(/https?:\/\/[^\s)]+/gi, '')
+    .replace(/^#+\s*/gm, '')
+    .replace(/\s{2,}/g, ' ')
+    .trim();
+}
+
+/**
+ * @param {string} fileName
+ * @returns {{ title: string, summary: string, body: string }}
+ */
+function parseMarkdownTopic(fileName) {
+  const raw = fs.readFileSync(path.join(HELP_MD_DIR, fileName), 'utf8');
+  const lines = raw.split(/\r?\n/);
+  let title = path.basename(fileName, '.md');
+  const bodyLines = [];
+  for (const line of lines) {
+    if (line.startsWith('# ')) {
+      title = stripMarkdown(line.slice(2));
+    } else if (line.trim()) {
+      bodyLines.push(line);
+    }
+  }
+  const body = stripMarkdown(bodyLines.join('\n'));
+  const summary = body.slice(0, 240);
+  return { title, summary, body };
+}
+
+/**
+ * @param {Object} row
+ * @returns {Object}
+ */
+function normalizeRow(row) {
+  const out = {
+    resourceType: String(row.resourceType || 'workspace').trim(),
+    title: String(row.title || '').trim(),
+    when: String(row.when || '').trim(),
+    expectedBehavior: String(row.expectedBehavior || '').trim(),
+    order: Number.isFinite(row.order) ? row.order : 100,
+    status: row.status === 'disabled' ? 'disabled' : 'active'
+  };
+  if (row.roleKey) {
+    out.roleKey = String(row.roleKey).trim();
+  }
+  if (row.description) {
+    out.description = String(row.description).trim();
+  }
+  return out;
+}
+
+/**
+ * @returns {Object[]}
+ */
+function rowsFromEvidencePatterns() {
+  const doc = JSON.parse(fs.readFileSync(EVIDENCE_PATTERNS, 'utf8'));
+  return (doc.patterns || []).map((p, i) =>
+    normalizeRow({
+      resourceType: p.resourceType || 'document',
+      title: p.title,
+      description: [p.summary, p.role ? `Audience: ${p.role}` : null, (p.steps || []).join(' → ')].filter(Boolean).join(' '),
+      when: p.summary || p.title,
+      expectedBehavior: (p.steps || []).join(' '),
+      order: 100 + i
+    })
+  );
+}
+
+/**
+ * @returns {Object[]}
+ */
+function rowsFromGoldenExamples() {
+  const rows = [];
+  let order = 200;
+  for (const file of fs.readdirSync(GOLDEN_DIR).sort()) {
+    if (!file.endsWith('.json')) {
+      continue;
+    }
+    const doc = JSON.parse(fs.readFileSync(path.join(GOLDEN_DIR, file), 'utf8'));
+    const traversal = (doc.metadataTraversal || []).join(', ');
+    rows.push(
+      normalizeRow({
+        resourceType: doc.resourceType || 'document',
+        title: `Golden example: ${doc.title || doc.key}`,
+        description: [doc.summary, doc.role ? `Audience: ${doc.role}` : null].filter(Boolean).join(' '),
+        when: `Designing a ${doc.resourceType || 'document'} integration similar to ${doc.key}`,
+        expectedBehavior: [
+          doc.summary,
+          doc.subscriptionGuidance,
+          traversal ? `Traverse sections: ${traversal}` : null,
+          (doc.recommendedSectionKeys || []).length
+            ? `Recommended sections: ${doc.recommendedSectionKeys.join(', ')}`
+            : null
+        ]
+          .filter(Boolean)
+          .join(' '),
+        order: order++
+      })
+    );
+  }
+  return rows;
+}
+
+/**
+ * @returns {Object[]}
+ */
+function rowsFromMarkdownTopics() {
+  return MD_TOPICS.map((t) => {
+    const parsed = parseMarkdownTopic(t.file);
+    return normalizeRow({
+      resourceType: t.resourceType,
+      title: parsed.title,
+      description: parsed.summary,
+      when: `Agent needs builder-api guidance for ${parsed.title}`,
+      expectedBehavior: parsed.body,
+      order: t.order
+    });
+  });
+}
+
+/**
+ * @returns {Object[]}
+ */
+function rowsFromPlatformRoles() {
+  const doc = JSON.parse(fs.readFileSync(PLATFORM_ROLES, 'utf8'));
+  const roleLines = (doc.roles || [])
+    .map((r) => `${r.value}: ${r.description}`)
+    .join(' ');
+  return [
+    normalizeRow({
+      resourceType: 'policy',
+      title: 'Platform RBAC roles for facade integrations',
+      description: 'Dataplane platform roles for orchestration facade systems (not per-system admin/reader).',
+      when: 'Patching system.rolesPermissions or repair rbac on a builder-api facade system',
+      expectedBehavior: `Use rbacPreset platform. Allowed roles: ${roleLines}. Avoid {systemKey}-admin or {systemKey}-reader on facade APIs.`,
+      order: 5
+    })
+  ];
+}
+
+/**
+ * @returns {Object[]}
+ */
+function rowsFromResourceCatalog() {
+  const doc = JSON.parse(fs.readFileSync(RESOURCE_CATALOG, 'utf8'));
+  const byCategory = {};
+  for (const rt of doc.resourceTypes || []) {
+    const cat = rt.category || 'other';
+    if (!byCategory[cat]) {
+      byCategory[cat] = [];
+    }
+    byCategory[cat].push(`${rt.key} (${rt.description})`);
+  }
+  const summary = Object.entries(byCategory)
+    .map(([cat, items]) => `${cat}: ${items.slice(0, 8).join('; ')}`)
+    .join(' ');
+  return [
+    normalizeRow({
+      resourceType: 'document',
+      title: 'Resource type catalog for datasource manifests',
+      description: 'Pick catalog keys that match business semantics; challenge generic placeholders.',
+      when: 'Choosing datasource.resourceType for a new or edited datasource',
+      expectedBehavior: `Compare purpose to catalog entries. Minimum system keys: ${(doc.minimumSystemKeys || []).join(', ')}. Categories: ${summary}`,
+      order: 8
+    })
+  ];
+}
+
+/**
+ * @returns {Object[]}
+ */
+function rowsFromManifestGuides() {
+  return [
+    normalizeRow({
+      resourceType: 'policy',
+      title: 'Manifest section keys and dependency order',
+      when: 'Agent edits external system or datasource manifest sections',
+      expectedBehavior:
+        'Read one section at a time via manifest section APIs. After metadata or capability patches, run repair hooks for fieldMappings, expose, and rbac. Use dependency graph: downstream sections derive from upstream patches.',
+      order: 15
+    }),
+    normalizeRow({
+      resourceType: 'workspace',
+      title: 'Repair hooks and CLI flag mapping',
+      when: 'Manifest repair is required after structural edits',
+      expectedBehavior:
+        'POST manifest repair with explicit hooks (fieldMappings, expose, rbac, sync, test). Align with repair CLI flags. Validate workspace before upload or certify.',
+      order: 16
+    }),
+    normalizeRow({
+      resourceType: 'workspace',
+      title: 'Section quality and missing inputs',
+      when: 'GET manifest section returns quality.missingInputs',
+      expectedBehavior:
+        'Collect required missing inputs from the user before PATCH. Do not guess values. Include sectionReadToken from GET in PATCH body.',
+      order: 17
+    })
+  ];
+}
+
+function main() {
+  const outPath = path.resolve(process.argv[2] || DEFAULT_OUT);
+  const rows = [
+    ...rowsFromPlatformRoles(),
+    ...rowsFromResourceCatalog(),
+    ...rowsFromMarkdownTopics(),
+    ...rowsFromManifestGuides(),
+    ...rowsFromEvidencePatterns(),
+    ...rowsFromGoldenExamples()
+  ];
+  fs.mkdirSync(path.dirname(outPath), { recursive: true });
+  fs.writeFileSync(outPath, `${JSON.stringify(rows, null, 2)}\n`, 'utf8');
+  process.stdout.write(`Wrote ${rows.length} evidence row(s) to ${outPath}\n`);
+}
+
+main();

package/scripts/lib/builder-api-docker-staging-copy.js

@@ -0,0 +1,60 @@
+/**
+ * Copy staged builder-api runtime tree to disk.
+ *
+ * @fileoverview Filesystem staging for Docker COPY layers
+ * @author AI Fabrix Team
+ * @version 1.0.0
+ */
+
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+
+/**
+ * @param {string} target
+ * @returns {void}
+ */
+function removeDirIfExists(target) {
+  if (fs.existsSync(target)) {
+    fs.rmSync(target, { recursive: true, force: true });
+  }
+}
+
+/**
+ * @param {string} src
+ * @param {string} dest
+ * @returns {void}
+ */
+function copyFileEnsureDir(src, dest) {
+  fs.mkdirSync(path.dirname(dest), { recursive: true });
+  fs.copyFileSync(src, dest);
+}
+
+/**
+ * @param {string} repoRoot
+ * @param {string} destination
+ * @param {string[]} manifest
+ * @returns {string[]}
+ */
+function stageFiles(repoRoot, destination, manifest) {
+  removeDirIfExists(destination);
+  fs.mkdirSync(destination, { recursive: true });
+
+  const staged = [];
+  for (const rel of manifest) {
+    const src = path.join(repoRoot, rel);
+    const dest = path.join(destination, rel);
+    if (!fs.existsSync(src)) {
+      continue;
+    }
+    copyFileEnsureDir(src, dest);
+    staged.push(rel);
+  }
+  return staged.sort();
+}
+
+module.exports = {
+  stageFiles,
+  removeDirIfExists
+};

package/scripts/lib/builder-api-docker-staging-core.js

@@ -0,0 +1,183 @@
+/**
+ * Runtime file discovery for builder-api Docker staging (plan 154.3).
+ *
+ * @fileoverview Require-graph walk + supplemental runtime assets
+ * @author AI Fabrix Team
+ * @version 1.0.0
+ */
+
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+
+const REQUIRE_RE = /require\s*\(\s*['"]([^'"]+)['"]\s*\)/g;
+
+/** @type {readonly string[]} */
+const ENTRY_FILES = ['bin/builder-api-spike.js', 'bin/builder-api-cleanup.js'];
+
+/** @type {readonly string[]} */
+const STAGING_SKIP_FILES = ['package.json', 'package-lock.json'];
+
+/** @type {readonly string[]} */
+const SUPPLEMENTAL_DIRS = [
+  'lib/schema',
+  'lib/programmatic/help-content',
+  'templates/openapi',
+  'templates/external-system',
+  'lib/datasource/capability/templates'
+];
+
+/** @type {readonly string[]} */
+const MANIFEST_FORBIDDEN_PREFIXES = [
+  'templates/applications/',
+  'templates/github/',
+  'templates/infra/',
+  'templates/python/',
+  'templates/typescript/',
+  'templates/channels/',
+  'integration/',
+  'tests/',
+  'scripts/',
+  '.cursor/',
+  'builder/',
+  'packages/',
+  'docs/',
+  'bin/aifabrix.js',
+  'lib/cli/'
+];
+
+/**
+ * @param {string} repoRoot
+ * @param {string} fromFile
+ * @param {string} req
+ * @returns {string|null}
+ */
+function resolveRelativeRequire(repoRoot, fromFile, req) {
+  if (!req.startsWith('.')) {
+    return null;
+  }
+  const base = path.dirname(fromFile);
+  let resolved = path.resolve(repoRoot, base, req);
+  if (fs.existsSync(resolved) && fs.statSync(resolved).isDirectory()) {
+    resolved = path.join(resolved, 'index.js');
+  }
+  if (!resolved.endsWith('.js') && fs.existsSync(`${resolved}.js`)) {
+    resolved = `${resolved}.js`;
+  }
+  if (!fs.existsSync(resolved)) {
+    return null;
+  }
+  return path.relative(repoRoot, resolved).split(path.sep).join('/');
+}
+
+/**
+ * @param {string} repoRoot
+ * @param {string} relFile
+ * @returns {string[]}
+ */
+function collectRequiresFromFile(repoRoot, relFile) {
+  const abs = path.join(repoRoot, relFile);
+  const content = fs.readFileSync(abs, 'utf8');
+  const found = [];
+  let match;
+  while ((match = REQUIRE_RE.exec(content)) !== null) {
+    const target = resolveRelativeRequire(repoRoot, relFile, match[1]);
+    if (target) {
+      found.push(target);
+    }
+  }
+  return found;
+}
+
+/**
+ * @param {string} dir
+ * @param {string} repoRoot
+ * @param {Set<string>} out
+ * @returns {void}
+ */
+function addDirectoryFiles(dir, repoRoot, out) {
+  const abs = path.join(repoRoot, dir);
+  if (!fs.existsSync(abs)) {
+    return;
+  }
+  for (const entry of fs.readdirSync(abs, { withFileTypes: true })) {
+    const rel = path.join(dir, entry.name).split(path.sep).join('/');
+    if (entry.isDirectory()) {
+      addDirectoryFiles(rel, repoRoot, out);
+      continue;
+    }
+    out.add(rel);
+  }
+}
+
+/**
+ * @param {string} repoRoot
+ * @returns {string[]}
+ */
+function discoverRuntimeFiles(repoRoot) {
+  const seen = new Set();
+  const queue = [...ENTRY_FILES];
+
+  while (queue.length > 0) {
+    const rel = queue.shift();
+    if (!rel || seen.has(rel)) {
+      continue;
+    }
+    if (!fs.existsSync(path.join(repoRoot, rel))) {
+      continue;
+    }
+    seen.add(rel);
+    for (const next of collectRequiresFromFile(repoRoot, rel)) {
+      if (!seen.has(next)) {
+        queue.push(next);
+      }
+    }
+  }
+
+  for (const dir of SUPPLEMENTAL_DIRS) {
+    addDirectoryFiles(dir, repoRoot, seen);
+  }
+
+  return [...seen].sort().filter((rel) => !STAGING_SKIP_FILES.includes(rel));
+}
+
+/**
+ * @param {string} relPath
+ * @returns {boolean}
+ */
+function isForbiddenManifestPath(relPath) {
+  const normalized = relPath.split(path.sep).join('/');
+  for (const prefix of MANIFEST_FORBIDDEN_PREFIXES) {
+    if (normalized === prefix.replace(/\/$/, '') || normalized.startsWith(prefix)) {
+      return true;
+    }
+  }
+  if (normalized.endsWith('.md') && normalized.startsWith('lib/')) {
+    if (normalized.startsWith('lib/programmatic/help-content/')) {
+      return false;
+    }
+    return true;
+  }
+  return false;
+}
+
+/**
+ * @param {string[]} manifest
+ * @returns {string[]}
+ */
+function findManifestViolations(manifest) {
+  return manifest.filter(isForbiddenManifestPath);
+}
+
+module.exports = {
+  ENTRY_FILES,
+  SUPPLEMENTAL_DIRS,
+  MANIFEST_FORBIDDEN_PREFIXES,
+  STAGING_SKIP_FILES,
+  discoverRuntimeFiles,
+  findManifestViolations,
+  isForbiddenManifestPath,
+  resolveRelativeRequire,
+  collectRequiresFromFile
+};