@scout9/app 1.0.0-alpha.0.1.9 → 1.0.0-alpha.0.1.91

package/src/core/config/agents.js

@@ -1,11 +1,94 @@
 import path from 'node:path';
-import { globSync } from 'glob/dist/esm/index.js';
+import fs from 'node:fs/promises';
+import colors from 'kleur';
+import { globSync } from 'glob';
+import { Configuration, Scout9Api } from '@scout9/admin';
 import { checkVariableType, requireProjectFile } from '../../utils/index.js';
-import { agentsConfigurationSchema } from '../../runtime/index.js';
+import { agentsBaseConfigurationSchema, agentsConfigurationSchema } from '../../runtime/index.js';
+import { audioExtensions } from '../../utils/audio-type.js';
+import { fileTypeFromBuffer } from '../../utils/file-type.js';
+import { videoExtensions } from '../../utils/video-type.js';
+import { projectTemplates } from '../../utils/project-templates.js';
+import { toBuffer, writeFileToLocal } from '../../utils/file.js';
+import imageBuffer from '../../utils/image-buffer.js';
+import audioBuffer from '../../utils/audio-buffer.js';
+import { yellow } from 'kleur/colors';
 
-export default async function loadAgentConfig({cwd = process.cwd(), folder = 'src'} = {}) {
-  console.log('@TODO implement loadAgentConfig');
-  const paths = globSync(path.resolve(cwd, `${folder}/entities/agents/{index,config}.{ts,js}`));
+
+async function registerAgent({agent}) {
+  if (agent.id) {
+    console.log(`Agent already registered: ${agent.id}`);
+    return agent.id;
+  }
+  const {id} = (await (new Scout9Api(new Configuration({apiKey: process.env.SCOUT9_API_KEY}))).agentRegister(
+    agent
+  ).then(res => res.data));
+  if (!id) {
+    throw new Error(`Failed to register agent`);
+  }
+  return id;
+}
+
+/**
+ * @param {Object} inputs
+ * @param {string | Buffer} inputs.img
+ * @param inputs.agentId
+ * @param [inputs.source='']
+ * @returns {Promise<string>}
+ */
+async function writeImgToServer({img, agentId, source = ''}) {
+  const {url} = (await (new Scout9Api(new Configuration({apiKey: process.env.SCOUT9_API_KEY}))).agentProfileUpload(
+    agentId,
+    await imageBuffer(img, source).then(r => r.buffer)
+  ).then(res => res.data));
+  if (!url) {
+    throw new Error(`Failed to upload agent image`);
+  }
+  return url;
+}
+
+async function writeTranscriptsToServer({transcripts, agentId, source = ''}) {
+  for (let i = 0; i < transcripts.length; i++) {
+    const result = await fileTypeFromBuffer(transcripts[i]);
+    if (!result || result.ext !== 'txt' || result.mime !== 'text/plain') {
+      throw new Error(`Invalid transcript type: ${result?.mime || 'N/A'}, expected text/plain (.txt file, got ${result?.ext || 'Missing ext'})`);
+    }
+  }
+  const {urls} = (await (new Scout9Api(new Configuration({apiKey: process.env.SCOUT9_API_KEY}))).agentTranscriptUpload(
+    agentId,
+    transcripts,
+  ).then(res => res.data));
+  if (!urls) {
+    throw new Error(`Failed to upload agent image`);
+  }
+  return urls;
+}
+
+async function writeAudiosToServer({audios, agentId, source = ''}) {
+  const buffers = [];
+  for (let i = 0; i < audios.length; i++) {
+    const {buffer} = await audioBuffer(audios[i], true, source);
+    buffers.push(buffer);
+  }
+  const {urls} = (await (new Scout9Api(new Configuration({apiKey: process.env.SCOUT9_API_KEY}))).agentTranscriptUpload(
+    agentId,
+    buffers
+  ).then(res => res.data));
+  if (!urls) {
+    throw new Error(`Failed to upload agent image`);
+  }
+  return urls;
+}
+
+export default async function loadAgentConfig({
+  cwd = process.cwd(),
+  dest = '/tmp/project',
+  deploying = false,
+  src = 'src',
+  cb = (message) => {
+  }
+} = {}) {
+  const paths = globSync(`${src}/entities/agents/{index,config}.{ts,js}`, {cwd, absolute: true});
   if (paths.length === 0) {
     throw new Error(`Missing required agents entity file, rerun "scout9 sync" to fix`);
   }
@@ -13,10 +96,12 @@ export default async function loadAgentConfig({cwd = process.cwd(), folder = 'sr
     throw new Error(`Multiple agents entity files found, rerun "scout9 sync" to fix`);
   }
 
+  const sourceFile = paths[0];
+
   /**
    * @type {Array<Agent> | function(): Array<Agent> | function(): Promise<Array<Agent>>}
    */
-  const mod = await requireProjectFile(paths[0]).then(mod => mod.default);
+  const mod = await requireProjectFile(sourceFile).then(mod => mod.default);
 
   // @type {Array<Agent>}
   let agents = [];
@@ -33,9 +118,217 @@ export default async function loadAgentConfig({cwd = process.cwd(), folder = 'sr
       throw new Error(`Invalid entity type (${entityType}) returned at "${path}"`);
   }
 
-  const result = agentsConfigurationSchema.safeParse(agents);
+  let serverDeployed = false; // Track whether we deployed to server, so that we can sync code base
+
+  // Send warnings if not properly registered
+  for (const agent of agents) {
+
+    if (!agent.id && deploying) {
+      const {img, transcripts, audios, ...rest} = agent;
+      agent.id = await registerAgent({agent: rest});
+      cb(`✅ Registered ${agent.firstName || 'agent'} with id: ${agent.id}`);
+      serverDeployed = true;
+    }
+
+    if (!agent.forwardPhone && !agent.forwardEmail) {
+      cb(yellow(`⚠️src/entities/agents.js|ts: neither a ".forwardPhone" or ".forwardEmail" to ${agent.firstName || JSON.stringify(agent)} - messages cannot be forward to you.`));
+    }
+    if (!agent.programmablePhoneNumber) {
+      const userName = agent.firstName ? `${agent.firstName}${agent.lastName ? ' ' + agent.lastName : ''}` : agent.forwardPhone;
+      cb(`⚠️${colors.yellow('Warning')}: ${userName} does not have a masked phone number to do auto replies. You can register one at ${colors.cyan(
+        'https://scout9.com/b')} under ${colors.green('users')} > ${colors.green(userName)}. Then run ${colors.cyan(
+        'scout9 sync')} to update.`);
+    }
+    if (agent.forwardPhone && agents.filter(a => a.forwardPhone && (a.forwardPhone === agent.forwardPhone)).length > 1) {
+      cb(yellow(`⚠️ src/entities/agents.js|ts: ".forwardPhone: ${agent.forwardPhone}" should only be associated to one agent within your project`));
+    }
+    if (agent.forwardEmail && agents.filter(a => a.forwardEmail && (a.forwardEmail === agent.forwardEmail)).length > 1) {
+      cb(yellow(`⚠️ src/entities/agents.js|ts: ".forwardEmail: ${agent.forwardEmail}" can only be associated to one agent within your project`));
+    }
+
+    // Handle agent image changes
+    if (agent.img) {
+      if (typeof agent.img === 'string') {
+        if (!agent.img.startsWith('https://storage.googleapis.com')) {
+          // Got string file, must either be a URL or a local file path
+          if (deploying) {
+            agent.img = await writeImgToServer({
+              img: agent.img,
+              agentId: agent.id,
+              source: sourceFile
+            });
+            cb(`✅ Uploaded ${agent.firstName || 'agent'}'s profile image to ${agent.img}`);
+            serverDeployed = true;
+          } else {
+            agent.img = await writeFileToLocal({
+              file: agent.img,
+              fileName: `${agent.firstName || 'agent'}.png`,
+              source: sourceFile
+            }).then(({uri, isImage}) => {
+              if (!isImage) {
+                throw new Error(`Invalid image type: ${typeof agent.img}`);
+              }
+              return uri;
+            });
+            cb(`✅ Copied ${agent.firstName || 'agent'}'s profile image to ${agent.img}`);
+          }
+        }
+      } else if (Buffer.isBuffer(agent.img)) {
+        if (deploying) {
+          agent.img = await writeImgToServer({
+            img: agent.img,
+            fileName: `${agent.firstName || 'agent'}.png`,
+            agentId: agent.id,
+            source: sourceFile
+          });
+          cb(`✅ Uploaded ${agent.firstName || 'agent'}'s profile image to ${agent.img}`);
+          serverDeployed = true;
+        } else {
+          agent.img = await writeFileToLocal({
+            file: agent.img,
+            fileName: `${agent.firstName || 'agent'}.png`,
+            source: sourceFile
+          }).then(({uri, isImage}) => {
+            if (!isImage) {
+              throw new Error(`Invalid image type: ${typeof agent.img}`);
+            }
+            return uri;
+          });
+          cb(`✅ Copied ${agent.firstName || 'agent'}'s profile image to ${agent.img}`);
+        }
+      } else {
+        throw new Error(`Invalid img type: ${typeof agent.img}`);
+      }
+    }
+
+
+    // Handle transcripts
+    if ((agent?.transcripts || []).length > 0) {
+      const deployedTranscripts = [];
+      const pendingTranscripts = [];
+      for (const transcript of agent.transcripts) {
+        if (typeof transcript === 'string') {
+          if (!transcript.startsWith('https://storage.googleapis.com')) {
+            pendingTranscripts.push(await fs.readFile(transcript));
+          } else {
+            deployedTranscripts.push(transcript);
+          }
+        } else if (Buffer.isBuffer(transcript)) {
+          const txtResult = await fileTypeFromBuffer(transcript);
+          if (txtResult?.ext !== 'txt' || txtResult?.mime !== 'text/plain') {
+            throw new Error(`Invalid transcript type: ${txtResult?.mime || 'N/A'}, expected text/plain (.txt file, got ${txtResult?.ext || 'Missing ext'})`);
+          }
+          pendingTranscripts.push(transcript);
+        } else {
+          throw new Error(`Invalid transcript type: ${typeof transcript}`);
+        }
+      }
+
+      let urls = [];
+      if (deploying) {
+        urls = await writeTranscriptsToServer({transcripts: pendingTranscripts, agentId: agent.id});
+        cb(`✅ Uploaded ${agent.firstName || 'agent'}'s transcripts to ${urls}`);
+        serverDeployed = true;
+      } else {
+        for (let i = 0; i < pendingTranscripts.length; i++) {
+          const transcript = pendingTranscripts[i];
+          urls.push(await writeFileToLocal({
+              file: transcript,
+              fileName: `transcript_${i + deployedTranscripts.length}.txt`,
+            source: sourceFile
+            }).then(({uri, mime, ext}) => {
+              if (mime !== 'text/plain') {
+                throw new Error(`Invalid transcript type: ${mime}, expected text/plain (.txt file, got ${ext})`);
+              }
+              return uri;
+            })
+          );
+        }
+        cb(`✅ Copied ${agent.firstName || 'agent'}'s transcripts to ${urls}`);
+      }
+
+      agent.transcripts = [
+        ...deployedTranscripts,
+        ...urls
+      ];
+    }
+
+    if ((agent?.audios || []).length > 0) {
+      const deployedAudios = [];
+      const pendingAudios = [];
+      for (const audio of agent.audios) {
+        if (typeof audio === 'string') {
+          if (!audio.startsWith('https://storage.googleapis.com')) {
+            // If not on GCS, then it must be a local file path or remote URL
+            pendingAudios.push(await toBuffer(audio, sourceFile).then(({buffer, ext, mime, isAudio, isVideo}) => {
+              if (!isAudio && !isVideo) {
+                throw new Error(`Invalid audio/video type: ${mime}, expected audio/* or video/* got ${ext}`);
+              }
+              return buffer;
+            }));
+          } else {
+            // Already deployed, append string URL
+            deployedAudios.push(audio);
+          }
+        } else if (Buffer.isBuffer(audio)) {
+          const fileType = await fileTypeFromBuffer(audio);
+          if (!fileType) {
+            throw new Error(`Invalid audio type: ${typeof audio}`);
+          }
+          if (videoExtensions.has(fileType.ext) || audioExtensions.has(fileType.ext)) {
+            pendingAudios.push(audio);
+          } else {
+            throw new Error(`Invalid audio/video type: ${fileType.mime}, expected audio/* or video/*, got ${fileType.ext}`);
+          }
+        } else {
+          throw new Error(`Invalid audio type: ${typeof audio}`);
+        }
+      }
+
+      let urls = [];
+      if (deploying) {
+        urls = await writeAudiosToServer({audios: pendingAudios, agentId: agent.id, source: sourceFile});
+        cb(`✅ Uploaded ${agent.firstName || 'agent'}'s audios to ${urls}`);
+        serverDeployed = true;
+      } else {
+        for (let i = 0; i < pendingAudios.length; i++) {
+          const audio = pendingAudios[i];
+          urls.push(await writeFileToLocal({
+            file: audio,
+            fileName: `audio_${i + deployedAudios.length}`,
+            source: sourceFile
+          }).then(({uri, mime, ext, isAudio, isVideo}) => {
+            if (!isAudio && !isVideo) {
+              throw new Error(`Invalid audio/video type: ${mime}, expected audio/* or video/* got ${ext}`);
+            }
+            return uri;
+          }));
+        }
+      }
+
+      agent.audios = [
+        ...deployedAudios,
+        ...urls
+      ];
+    }
+  }
+
+  const result = (deploying ? agentsConfigurationSchema : agentsBaseConfigurationSchema).safeParse(agents);
   if (!result.success) {
+    result.error.source = paths[0];
     throw result.error;
   }
+
+  if (serverDeployed) {
+    cb(`Syncing ${sourceFile} with latest server changes`);
+    await fs.writeFile(sourceFile, projectTemplates.entities.agents(agents, path.extname(sourceFile)));
+    // const update = await p.confirm({
+    //   message: `Changes uploaded, sync local entities/agents file?`,
+    //   initialValue: true
+    // });
+    // if (update) {
+    // }
+  }
+
   return agents;
 }

package/src/core/config/entities.js

@@ -1,21 +1,19 @@
 import { globSync } from 'glob';
 import path from 'node:path';
 import {
-  entitiesRootProjectConfigurationSchema,
-  entityApiConfigurationSchema,
-  entityConfigurationSchema,
-  entityRootProjectConfigurationSchema
+    entitiesRootProjectConfigurationSchema,
+    entityApiConfigurationSchema,
+    entityConfigurationSchema,
+    entityRootProjectConfigurationSchema
 } from '../../runtime/index.js';
 import { checkVariableType, requireOptionalProjectFile, requireProjectFile } from '../../utils/index.js';
+import { logUserValidationError } from '../../report.js';
 
 async function loadEntityApiConfig(cwd, filePath) {
   const dir = path.dirname(filePath);
   const extension = path.extname(filePath);
-  const apiFilePath = path.join(dir, `api${extension}`);
-  const root = cwd.split('/').pop();
-  const x = apiFilePath.replace(cwd, '').split('/').slice(1).join('/');
-
-  const mod = await requireOptionalProjectFile(x);
+  const apiFilePath = path.resolve(dir, `api${extension}`);
+  const mod = await requireOptionalProjectFile(apiFilePath);
 
   if (mod) {
     const config = {};
@@ -32,34 +30,40 @@ async function loadEntityApiConfig(cwd, filePath) {
   }
 }
 
+/**
+ * @returns {Promise<import('../../runtime/client/config.js').IEntitiesRootProjectConfiguration>}
+ */
 export default async function loadEntitiesConfig(
-  {cwd = process.cwd(), folder = 'src'} = {}
+  {cwd = process.cwd(), src = 'src', logger, cb = (message) => {}} = {}
 ) {
-  console.log('loadEntitiesConfig', {cwd, folder});
+  /** @type import('../../runtime/client/config.js').IEntitiesRootProjectConfiguration */
   const config = [];
-  const paths = globSync(path.resolve(cwd, `${folder}/entities/**/{index,config,api}.{ts,js}`));
+  // const paths = globSync(path.resolve(cwd, `${src}/entities/**/{index,config,api}.{ts,js}`), {cwd, absolute: true});
+  const filePaths = globSync(`${src}/entities/**/{index,config,api}.{ts,js}`, {cwd, absolute: true});
   const data = [];
-  for (const path of paths) {
-    const segments = path.split('/');
-    const srcIndex = segments.findIndex((segment, index) => segment === folder && segments[index + 1] === 'entities');
-    const parents = segments.slice(srcIndex + 2, -1).reverse(); // +2 to skip "${folder}" and "entities"
+  for (const filePath of filePaths) {
+    const segments = filePath.split(path.sep); // Use path.sep for platform independence
+    // const segments = filePath.split('/');
+    const srcIndex = segments.findIndex((segment, index) => segment === src && segments[index + 1] === 'entities');
+    const parents = segments.slice(srcIndex + 2, -1).reverse(); // +2 to skip "${src}" and "entities"
     if (parents.length > 0) {
-      const api = await loadEntityApiConfig(cwd, path);
-      data.push({path, parents, api});
+      const api = await loadEntityApiConfig(cwd, filePath);
+      data.push({filePath, parents, api});
     } else {
-      console.log(`WARNING: "${path}" Is not a valid entity path, must be contained in a named folder under entities/`);
+      console.log(`WARNING: "${filePath}" Is not a valid entity filePath, must be contained in a named src under entities/`);
     }
   }
 
   const specialEntities = ['agents'];
 
-  for (const {path, parents, api} of data) {
+  for (const {filePath, parents, api} of data) {
     let entityConfig = {};
-    const fileName = path.split('/')?.pop()?.split('.')?.[0];
-    if (!fileName) throw new Error(`Invalid file name "${path}"`);
+    // const fileName = filePath.split('/')?.pop()?.split('.')?.[0];
+    const fileName = path.basename(filePath, path.extname(filePath));
+    if (!fileName) throw new Error(`Invalid file name "${filePath}"`);
     const isSpecial = specialEntities.some(se => parents.includes(se));
     if ((fileName === 'index' || fileName === 'config') && !isSpecial) {
-      const entityConfigHandler = await requireProjectFile(path).then(mod => mod.default);
+      const entityConfigHandler = await requireProjectFile(filePath).then(mod => mod.default);
       // Check if entityConfig is a function or object
       const entityType = checkVariableType(entityConfigHandler);
       switch (entityType) {
@@ -71,12 +75,13 @@ export default async function loadEntitiesConfig(
           entityConfig = entityConfigHandler;
           break;
         default:
-          throw new Error(`Invalid entity type (${entityType}) returned at "${path}"`);
+          throw new Error(`Invalid entity type (${entityType}) returned at "${filePath}"`);
       }
 
       // Validate entity configuration
       const result = entityConfigurationSchema.safeParse(entityConfig, {path: ['entities', config.length]});
       if (!result.success) {
+        logUserValidationError(result.error, filePath);
         throw result.error;
       }
     } else if (isSpecial && (fileName === 'index' || fileName === 'config')) {
@@ -88,16 +93,41 @@ export default async function loadEntitiesConfig(
     const entityProjectConfig = {
       ...entityConfig,
       entity: parents[0],
-      entities: parents,
+      entities: parents.reverse(),
       api
     };
     entityRootProjectConfigurationSchema.parse(entityProjectConfig);
-
-    config.push(entityProjectConfig);
+    const existingIndex = config.findIndex(c => c.entity === entityProjectConfig.entity);
+    if (existingIndex > -1) {
+      if (config[existingIndex].entities.length !== entityProjectConfig.entities.length) {
+        throw new Error(`Invalid entity configuration at "${filePath}", entity name mismatch`);
+      }
+        config[existingIndex] = {
+          ...config[existingIndex],
+          definitions: [
+              ...(config[existingIndex].definitions),
+                ...(entityProjectConfig.definitions || [])
+          ],
+          training: [
+            ...(config[existingIndex].training),
+            ...(entityProjectConfig.training || [])
+          ],
+          tests: [
+            ...(config[existingIndex].tests),
+            ...(entityProjectConfig.tests || [])
+          ],
+            api: (!config[existingIndex].api && !entityProjectConfig.api) ? null : {
+            ...(config[existingIndex].api || {}),
+            ...(entityProjectConfig.api || {})
+          }
+        }
+    } else {
+      config.push(entityProjectConfig);
+    }
   }
 
   if (!config.some(c => c.entity === 'customers')) {
-    throw new Error(`Missing required entity: "entities/customers"`);
+    throw new Error(`Missing required entity: "entities/customers" in ${src}`);
   }
   if (!config.some(c => c.entity === '[customer]')) {
     throw new Error(`Missing required entity: "entities/customers/[customer]"`);

package/src/core/config/index.js

@@ -6,10 +6,17 @@ import loadEntitiesConfig from './entities.js';
 import loadProjectConfig from './project.js';
 import loadWorkflowsConfig from './workflow.js';
 import { Scout9ProjectBuildConfigSchema } from '../../runtime/index.js';
+import { ProgressLogger } from '../../utils/index.js';
 
 
-export function loadEnvConfig({ cwd = process.cwd()} = {}) {
+export function loadEnvConfig({
+  cwd = process.cwd(), cb = (msg) => {
+  }
+} = {}) {
   if (!!process.env.SCOUT9_API_KEY) {
+    if (process.env.SCOUT9_API_KEY.includes('insert-scout9-api-key')) {
+      throw new Error('Missing SCOUT9_API_KEY, please add your Scout9 API key to your .env file');
+    }
     return;
   }
   const configFilePath = path.resolve(cwd, './.env');
@@ -17,31 +24,46 @@ export function loadEnvConfig({ cwd = process.cwd()} = {}) {
   if (!process.env.SCOUT9_API_KEY) {
     const exists = fs.existsSync(configFilePath);
     if (!exists) {
-      throw new Error(`Missing .env file with SCOUT9_API_KEY`);
+      throw new Error(`Missing .env file with "SCOUT9_API_KEY".\n\n\tTo fix, create a .env file at the root of your project.\nAdd "SCOUT9_API_KEY=<your-scout9-api-key>" to the .env file.\n\n\t> You can get your API key at https://scout9.com\n\n`);
     } else {
-      throw new Error('Missing SCOUT9_API_KEY within .env file');
+      throw new Error(`Missing "SCOUT9_API_KEY" within .env file.\n\n\tTo fix, add "SCOUT9_API_KEY=<your-scout9-api-key>" to the .env file.\n\n\tYou can get your API key at https://scout9.com\n\n`);
     }
   }
 }
 
 /**
- *
- * @param cwd
- * @param folder
- * @returns {Promise<Scout9ProjectBuildConfig>}
+ * @deprecated use "new ProjectFiles(...).load()" instead
+ * @param {{cwd: string; src: string; logger?: ProgressLogger; deploying?: boolean; cb?: (message: string) => void}} - build options
+ * @returns {Promise<import('../../runtime/client/config.js').IScout9ProjectBuildConfig>}
  */
-export async function loadConfig({ cwd = process.cwd(), folder = 'src'} = {}) {
+export async function loadConfig({
+  cwd = process.cwd(), src = 'src', dest = '/tmp/project', deploying = false, logger = new ProgressLogger(), cb = (msg) => {
+  }
+} = {}) {
   // Load globals
-  loadEnvConfig({cwd});
+  loadEnvConfig({cwd, src, logger, cb});
+
+  const baseProjectConfig = await loadProjectConfig({cwd, src, logger, cb, deploying});
+  const entitiesConfig = await loadEntitiesConfig({cwd, src, logger, cb, deploying});
+  const agentsConfig = await loadAgentConfig({cwd, src, logger, cb, deploying, dest});
+  const workflowsConfig = await loadWorkflowsConfig({cwd, src, logger, deploying, cb});
+
+  /**
+   * @type {import('../../runtime/client/config.js').IScout9ProjectBuildConfig}
+   */
   const projectConfig = {
-    ...await loadProjectConfig({cwd, folder}),
-    entities: await loadEntitiesConfig({cwd, folder}),
-    agents: await loadAgentConfig({cwd, folder}),
-    workflows: await loadWorkflowsConfig({cwd, folder})
-  }
+    ...baseProjectConfig,
+    entities: entitiesConfig,
+    agents: agentsConfig,
+    workflows: workflowsConfig
+  };
 
   // Validate the config
-  Scout9ProjectBuildConfigSchema.parse(projectConfig);
+  const result = Scout9ProjectBuildConfigSchema.safeParse(projectConfig);
+  if (!result.success) {
+    result.error.source = `${src}/index.js`;
+    throw result.error;
+  }
 
   return projectConfig;
 }