@aifabrix/builder 2.51.0 → 2.51.1

package/lib/role-assistant/knowledge-sync.js

@@ -0,0 +1,465 @@
+/**
+ * Role Assistant knowledge import/export for document storage datasources.
+ *
+ * @fileoverview Sync local role-assistant knowledge/*.md with dataplane document storage
+ * @author AI Fabrix Team
+ * @version 1.0.0
+ */
+
+'use strict';
+
+const path = require('path');
+const crypto = require('crypto');
+const fs = require('../internal/fs-real-sync');
+const yaml = require('js-yaml');
+const logger = require('../utils/logger');
+const { formatSuccessLine, formatWarningLine, metadata } = require('../utils/cli-test-layout-chalk');
+const { listDocuments, validateDocuments, bulkDocuments } = require('../api/datasources-extended.api');
+const {
+  extractDocumentStorageKeyFromAssistantDetails,
+  getAssistantDetails,
+  updateAssistantSettings
+} = require('../api/assistants.api');
+const { resolveExternalSystemPath } = require('../utils/app-path-resolve');
+const {
+  SYSTEM_KIND_DIGITAL_WORKER,
+  getRoleAssistantYamlPath,
+  readRoleAssistantSettings,
+  resolveDocumentStorageKeyFromRoleAssistantSettings,
+  resolveRoleAssistantEnvironmentSettings
+} = require('./settings');
+
+function resolveSystemNode(input) {
+  if (!input || typeof input !== 'object') {
+    return null;
+  }
+  return input.system || input.application || null;
+}
+
+function isDigitalWorkerModel(input) {
+  const systemNode = resolveSystemNode(input);
+  if (!systemNode || typeof systemNode.systemKind !== 'string') {
+    return false;
+  }
+  return systemNode.systemKind.trim() === SYSTEM_KIND_DIGITAL_WORKER;
+}
+
+function resolveDatasourceKeyFromCollection(dataSources) {
+  if (!Array.isArray(dataSources)) {
+    return null;
+  }
+  for (const row of dataSources) {
+    if (!row || typeof row !== 'object') {
+      continue;
+    }
+    if (String(row.entityType || '').trim() !== 'documentStorage') {
+      continue;
+    }
+    const candidate = String(row.key || row.datasourceKey || '').trim();
+    if (candidate) {
+      return candidate;
+    }
+  }
+  return null;
+}
+
+async function resolveDocumentStorageKeyFromDataplane(dataplaneUrl, authConfig, systemKey) {
+  if (!dataplaneUrl || !authConfig || !systemKey) {
+    return null;
+  }
+  try {
+    const response = await getAssistantDetails(dataplaneUrl, systemKey, authConfig);
+    return extractDocumentStorageKeyFromAssistantDetails(response);
+  } catch (_error) {
+    return null;
+  }
+}
+
+/**
+ * Push role-assistant.yaml knowledge.documentStorageKey to dataplane Assistants settings.
+ * Called on upload/deploy so the operator UI and CLI stay aligned.
+ */
+async function pushRoleAssistantDocumentStorageSettings(params) {
+  const {
+    systemKey,
+    dataplaneUrl,
+    authConfig,
+    environment = 'dev',
+    pathOptions = {},
+    dryRun = false
+  } = params;
+  const settingsEnv = resolveRoleAssistantSettingsEnvKey(environment);
+  const settings = readRoleAssistantSettings(systemKey, { required: false, pathOptions });
+  if (!settings) {
+    return { skipped: true, reason: 'missing-role-assistant-settings' };
+  }
+  const section = resolveRoleAssistantEnvironmentSettings(settings, settingsEnv, {
+    required: false
+  });
+  if (!section) {
+    return { skipped: true, reason: 'missing-env-settings' };
+  }
+  const documentStorageKey =
+    resolveDocumentStorageKeyFromRoleAssistantSettings(settings, settingsEnv) || '';
+  if (dryRun) {
+    return {
+      skipped: false,
+      dryRun: true,
+      documentStorageKey: documentStorageKey || null
+    };
+  }
+  if (!dataplaneUrl || !authConfig) {
+    return { skipped: true, reason: 'missing-dataplane-auth' };
+  }
+  const response = await updateAssistantSettings(dataplaneUrl, systemKey, authConfig, {
+    documentStorageKey
+  });
+  if (response && response.success === false) {
+    throw new Error(
+      response.formattedError ||
+        response.error ||
+        'Failed to update Knowledge document storage settings'
+    );
+  }
+  return {
+    skipped: false,
+    documentStorageKey: documentStorageKey || null
+  };
+}
+
+/**
+ * Resolve Knowledge document storage key.
+ * Order: role-assistant.yaml → dataplane Assistants settings → package dataSources.
+ */
+async function resolveRoleAssistantDocumentStorageKey({
+  systemKey,
+  model,
+  environment,
+  pathOptions,
+  dataplaneUrl,
+  authConfig
+}) {
+  const settingsEnv = resolveRoleAssistantSettingsEnvKey(environment);
+  const settings = readRoleAssistantSettings(systemKey, { required: false, pathOptions });
+  const settingsKey = resolveDocumentStorageKeyFromRoleAssistantSettings(settings, settingsEnv);
+  if (settingsKey) {
+    return settingsKey;
+  }
+  const fromApi = await resolveDocumentStorageKeyFromDataplane(
+    dataplaneUrl,
+    authConfig,
+    systemKey
+  );
+  if (fromApi) {
+    return fromApi;
+  }
+  return resolveDatasourceKeyFromCollection(model && model.dataSources);
+}
+
+function collectKnowledgeMarkdownFiles(dirPath) {
+  if (!fs.existsSync(dirPath)) {
+    return [];
+  }
+  const entries = fs.readdirSync(dirPath, { withFileTypes: true });
+  const files = [];
+  for (const entry of entries) {
+    const fullPath = path.join(dirPath, entry.name);
+    if (entry.isDirectory()) {
+      files.push(...collectKnowledgeMarkdownFiles(fullPath));
+      continue;
+    }
+    const lower = entry.name.toLowerCase();
+    if (lower.endsWith('.md') || lower.endsWith('.markdown')) {
+      files.push(fullPath);
+    }
+  }
+  return files;
+}
+
+function normalizeDocumentExternalId(knowledgeRoot, filePath) {
+  const relative = path.relative(knowledgeRoot, filePath).replace(/\\/g, '/');
+  return relative.replace(/\.(md|markdown)$/i, '');
+}
+
+function buildDocumentRecord(knowledgeRoot, filePath) {
+  const content = fs.readFileSync(filePath, 'utf8');
+  const externalId = normalizeDocumentExternalId(knowledgeRoot, filePath);
+  const title = path.basename(externalId);
+  const contentHash = crypto.createHash('sha256').update(content, 'utf8').digest('hex');
+  const metadata = {
+    externalId,
+    title,
+    mimeType: 'text/markdown',
+    sourcePath: path.relative(knowledgeRoot, filePath).replace(/\\/g, '/'),
+    sourceType: 'role-assistant-markdown'
+  };
+  return {
+    key: externalId,
+    name: title,
+    contentHash,
+    metadata,
+    binaryBase64: Buffer.from(content, 'utf8').toString('base64'),
+    // retained for export/debug consumers that still read flat fields
+    externalId,
+    title,
+    content
+  };
+}
+
+function toDocumentValidatePayload(record) {
+  return {
+    key: record.key,
+    contentHash: record.contentHash,
+    metadata: record.metadata || {}
+  };
+}
+
+function toDocumentBulkPayload(record) {
+  return {
+    key: record.key,
+    name: record.name || record.title || record.key,
+    contentHash: record.contentHash,
+    metadata: record.metadata || {},
+    binaryBase64: record.binaryBase64
+  };
+}
+
+function extractDocumentRows(payload) {
+  const candidates = [
+    payload?.data?.items,
+    payload?.data?.rows,
+    payload?.data,
+    payload?.items,
+    payload?.rows
+  ];
+  for (const candidate of candidates) {
+    if (Array.isArray(candidate)) {
+      return candidate;
+    }
+  }
+  return [];
+}
+
+function extractDocumentContent(row) {
+  const values = [row?.content, row?.text, row?.body, row?.markdown, row?.document];
+  for (const value of values) {
+    if (typeof value === 'string') {
+      return value;
+    }
+  }
+  return '';
+}
+
+function resolveDocumentFileName(row, index) {
+  const seed = row?.externalId || row?.documentKey || row?.key || row?.id || row?.title || `doc-${index + 1}`;
+  const normalized = String(seed).trim().replace(/[^\w.-/]/g, '-').replace(/\/+/g, '/');
+  const filePath = normalized.endsWith('.md') ? normalized : `${normalized}.md`;
+  return filePath.startsWith('/') ? filePath.slice(1) : filePath;
+}
+
+function resolveRoleAssistantSettingsEnvKey(environment) {
+  const token = String(environment || '').trim().toLowerCase();
+  return token === 'dev' || token === 'tst' || token === 'pro' ? token : 'dev';
+}
+
+function writeRoleAssistantSettingsFile(systemKey, docStorageKey, environment = 'dev', pathOptions = {}) {
+  const roleAssistantPath = resolveExternalSystemPath(systemKey, pathOptions);
+  const roleAssistantYamlPath = getRoleAssistantYamlPath(systemKey, { pathOptions });
+  if (fs.existsSync(roleAssistantYamlPath)) {
+    return roleAssistantYamlPath;
+  }
+  const envKey = resolveRoleAssistantSettingsEnvKey(environment);
+  fs.mkdirSync(roleAssistantPath, { recursive: true });
+  const payload = {
+    apiVersion: 'aifabrix.ai/v1',
+    version: '1.0.0',
+    kind: 'RoleAssistantSettings',
+    roleKey: systemKey,
+    [envKey]: {
+      knowledge: docStorageKey ? { documentStorageKey: docStorageKey } : {}
+    }
+  };
+  fs.writeFileSync(roleAssistantYamlPath, yaml.dump(payload, { lineWidth: -1 }), 'utf8');
+  return roleAssistantYamlPath;
+}
+
+async function uploadRoleAssistantKnowledgeRecords(dataplaneUrl, datasourceKey, authConfig, records) {
+  const validateBody = records.map(toDocumentValidatePayload);
+  const validationResult = await validateDocuments(
+    dataplaneUrl,
+    datasourceKey,
+    authConfig,
+    validateBody
+  );
+  if (!validationResult || validationResult.success === false) {
+    throw new Error(validationResult?.formattedError || validationResult?.error || 'Knowledge validation failed');
+  }
+  const bulkBody = records.map(toDocumentBulkPayload);
+  const uploadResult = await bulkDocuments(dataplaneUrl, datasourceKey, authConfig, bulkBody);
+  if (!uploadResult || uploadResult.success === false) {
+    throw new Error(uploadResult?.formattedError || uploadResult?.error || 'Knowledge upload failed');
+  }
+  const payload = uploadResult.data && typeof uploadResult.data === 'object' ? uploadResult.data : uploadResult;
+  return Number(payload.inserted || payload.upserted || payload.recordsProcessed || bulkBody.length || 0);
+}
+
+async function importRoleAssistantKnowledge(params) {
+  const {
+    systemKey,
+    dataplaneUrl,
+    authConfig,
+    model,
+    environment = 'dev',
+    pathOptions = {},
+    dryRun = false
+  } = params;
+  if (!isDigitalWorkerModel(model)) {
+    return { skipped: true, reason: 'not-role-assistant' };
+  }
+  const datasourceKey = await resolveRoleAssistantDocumentStorageKey({
+    systemKey,
+    model,
+    environment,
+    pathOptions,
+    dataplaneUrl,
+    authConfig
+  });
+  if (!datasourceKey) {
+    return { skipped: true, reason: 'missing-document-storage-datasource' };
+  }
+  const appPath = resolveExternalSystemPath(systemKey, pathOptions);
+  const knowledgePath = path.join(appPath, 'knowledge');
+  const files = collectKnowledgeMarkdownFiles(knowledgePath);
+  const records = files.map((filePath) => buildDocumentRecord(knowledgePath, filePath));
+  if (records.length === 0) {
+    return { skipped: true, reason: 'no-knowledge-files', datasourceKey };
+  }
+  if (dryRun) {
+    return { skipped: false, dryRun: true, datasourceKey, fileCount: files.length, recordCount: records.length };
+  }
+  const uploaded = await uploadRoleAssistantKnowledgeRecords(
+    dataplaneUrl,
+    datasourceKey,
+    authConfig,
+    records
+  );
+  return {
+    skipped: false,
+    datasourceKey,
+    fileCount: files.length,
+    recordCount: records.length,
+    uploaded
+  };
+}
+
+async function exportRoleAssistantKnowledge(params) {
+  const { systemKey, dataplaneUrl, authConfig, manifest, environment = 'dev', pathOptions = {} } = params;
+  const model = { system: manifest?.application, dataSources: manifest?.dataSources || [] };
+  if (!isDigitalWorkerModel(model)) {
+    return { skipped: true, reason: 'not-role-assistant' };
+  }
+  const datasourceKey = await resolveRoleAssistantDocumentStorageKey({
+    systemKey,
+    model,
+    environment,
+    pathOptions,
+    dataplaneUrl,
+    authConfig
+  });
+  if (!datasourceKey) {
+    const settingsPath = writeRoleAssistantSettingsFile(systemKey, null, environment, pathOptions);
+    return { skipped: true, reason: 'missing-document-storage-datasource', settingsPath };
+  }
+  const appPath = resolveExternalSystemPath(systemKey, pathOptions);
+  const knowledgePath = path.join(appPath, 'knowledge');
+  fs.mkdirSync(knowledgePath, { recursive: true });
+  const response = await listDocuments(dataplaneUrl, datasourceKey, authConfig, { limit: 1000 });
+  if (!response || response.success === false) {
+    throw new Error(response?.formattedError || response?.error || 'Knowledge export failed');
+  }
+  const rows = extractDocumentRows(response);
+  rows.forEach((row, index) => {
+    const relativeFile = resolveDocumentFileName(row, index);
+    const fullPath = path.join(knowledgePath, relativeFile);
+    fs.mkdirSync(path.dirname(fullPath), { recursive: true });
+    fs.writeFileSync(fullPath, `${extractDocumentContent(row)}\n`, 'utf8');
+  });
+  const settingsPath = writeRoleAssistantSettingsFile(systemKey, datasourceKey, environment, pathOptions);
+  return { skipped: false, datasourceKey, count: rows.length, knowledgePath, settingsPath };
+}
+
+function logKnowledgeImportSummary(systemKey, result, options = {}) {
+  if (result.skipped) {
+    if (options.verbose === true) {
+      logger.log(metadata(`Knowledge sync skipped (${systemKey}): ${result.reason}`));
+    }
+    return;
+  }
+  if (result.dryRun) {
+    logger.log(
+      formatWarningLine(
+        `Role assistant knowledge dry-run: ${result.fileCount} file(s), ${result.recordCount} record(s), datasource ${result.datasourceKey}`
+      )
+    );
+    return;
+  }
+  logger.log(
+    formatSuccessLine(
+      `Role assistant knowledge synced (${result.datasourceKey}): ${result.recordCount} record(s) from ${result.fileCount} file(s).`
+    )
+  );
+}
+
+function logDocumentStorageSettingsPushSummary(systemKey, result, options = {}) {
+  if (result.skipped) {
+    if (options.verbose === true) {
+      logger.log(
+        metadata(`Knowledge document storage settings skipped (${systemKey}): ${result.reason}`)
+      );
+    }
+    return;
+  }
+  if (result.dryRun) {
+    logger.log(
+      formatWarningLine(
+        `Knowledge document storage settings dry-run (${systemKey}): ${
+          result.documentStorageKey || '(clear)'
+        }`
+      )
+    );
+    return;
+  }
+  logger.log(
+    formatSuccessLine(
+      `Knowledge document storage settings updated (${systemKey}): ${
+        result.documentStorageKey || '(cleared)'
+      }`
+    )
+  );
+}
+
+function logKnowledgeExportSummary(systemKey, result, options = {}) {
+  if (result.skipped) {
+    if (options.verbose === true) {
+      logger.log(metadata(`Knowledge export skipped (${systemKey}): ${result.reason}`));
+    }
+    return;
+  }
+  logger.log(
+    formatSuccessLine(
+      `Role assistant knowledge exported (${result.datasourceKey}): ${result.count} document(s) to ${result.knowledgePath}.`
+    )
+  );
+}
+
+module.exports = {
+  isDigitalWorkerModel,
+  resolveRoleAssistantSettingsEnvKey,
+  resolveRoleAssistantDocumentStorageKey,
+  pushRoleAssistantDocumentStorageSettings,
+  importRoleAssistantKnowledge,
+  exportRoleAssistantKnowledge,
+  logKnowledgeImportSummary,
+  logDocumentStorageSettingsPushSummary,
+  logKnowledgeExportSummary
+};

package/lib/role-assistant/settings.js

@@ -0,0 +1,126 @@
+/**
+ * Role Assistant settings reader and validator.
+ *
+ * @fileoverview role-assistant.yaml parsing for digitalWorker packages
+ * @author AI Fabrix Team
+ * @version 1.0.0
+ */
+
+'use strict';
+
+const path = require('path');
+const fs = require('../internal/fs-real-sync');
+const yaml = require('js-yaml');
+const { resolveExternalSystemPath } = require('../utils/app-path-resolve');
+const { loadRoleAssistantSettingsSchema } = require('../utils/schema-loader');
+const { SYSTEM_KIND_DIGITAL_WORKER } = require('../utils/role-assistant-paths');
+
+function getRoleAssistantYamlPath(systemKey, options = {}) {
+  const appPath = resolveExternalSystemPath(systemKey, options.pathOptions || {});
+  const yamlPath = path.join(appPath, 'role-assistant.yaml');
+  if (fs.existsSync(yamlPath)) {
+    return yamlPath;
+  }
+  const ymlPath = path.join(appPath, 'role-assistant.yml');
+  if (fs.existsSync(ymlPath)) {
+    return ymlPath;
+  }
+  return yamlPath;
+}
+
+function isDigitalWorkerManifest(manifest) {
+  return Boolean(
+    manifest &&
+      manifest.system &&
+      typeof manifest.system.systemKind === 'string' &&
+      manifest.system.systemKind.trim() === SYSTEM_KIND_DIGITAL_WORKER
+  );
+}
+
+function parseRoleAssistantYaml(roleAssistantPath) {
+  const content = fs.readFileSync(roleAssistantPath, 'utf8');
+  const parsed = yaml.load(content);
+  if (!parsed || typeof parsed !== 'object') {
+    throw new Error(`Invalid role-assistant.yaml content: ${roleAssistantPath}`);
+  }
+  return parsed;
+}
+
+function formatAjvErrors(errors) {
+  if (!Array.isArray(errors) || errors.length === 0) {
+    return ['role-assistant.yaml schema validation failed'];
+  }
+  return errors.map((item) => {
+    const location = item.instancePath || '(root)';
+    return `${location} ${item.message}`;
+  });
+}
+
+function validateRoleAssistantSettings(settings, roleAssistantPath) {
+  const validate = loadRoleAssistantSettingsSchema();
+  const valid = validate(settings);
+  if (!valid) {
+    const lines = formatAjvErrors(validate.errors).join('; ');
+    throw new Error(`Invalid role-assistant.yaml (${roleAssistantPath}): ${lines}`);
+  }
+}
+
+function readRoleAssistantSettings(systemKey, options = {}) {
+  const filePath = getRoleAssistantYamlPath(systemKey, options);
+  if (!fs.existsSync(filePath)) {
+    if (options.required === true) {
+      throw new Error(`Missing required role-assistant.yaml: ${filePath}`);
+    }
+    return null;
+  }
+  const settings = parseRoleAssistantYaml(filePath);
+  validateRoleAssistantSettings(settings, filePath);
+  return settings;
+}
+
+function resolveRoleAssistantEnvironmentSettings(settings, envName, options = {}) {
+  if (!settings || typeof settings !== 'object') {
+    return null;
+  }
+  const env = String(envName || '').trim() || 'dev';
+  const section = settings[env];
+  if (!section) {
+    if (options.required !== false) {
+      throw new Error(
+        `role-assistant.yaml is missing "${env}" section. Add "${env}" with runtime/debug/knowledge settings.`
+      );
+    }
+    return null;
+  }
+  return section;
+}
+
+function resolveDocumentStorageKeyFromRoleAssistantSettings(settings, envName) {
+  const section = resolveRoleAssistantEnvironmentSettings(settings, envName, { required: false });
+  const key = section && section.knowledge && section.knowledge.documentStorageKey;
+  if (typeof key !== 'string' || !key.trim()) {
+    return null;
+  }
+  return key.trim();
+}
+
+function resolveRoleAssistantSettingsVersion(settings) {
+  if (!settings || typeof settings !== 'object') {
+    return null;
+  }
+  const value = settings.version;
+  if (typeof value !== 'string' || !value.trim()) {
+    return null;
+  }
+  return value.trim();
+}
+
+module.exports = {
+  SYSTEM_KIND_DIGITAL_WORKER,
+  getRoleAssistantYamlPath,
+  isDigitalWorkerManifest,
+  readRoleAssistantSettings,
+  resolveRoleAssistantEnvironmentSettings,
+  resolveDocumentStorageKeyFromRoleAssistantSettings,
+  resolveRoleAssistantSettingsVersion
+};