@scout9/app 1.0.0-alpha.0.1.9 → 1.0.0-alpha.0.1.90

package/src/core/sync.js

@@ -0,0 +1,71 @@
+import { platformApi } from './data.js';
+
+/**
+ * @param {import('../runtime/client/config.js').IScout9ProjectBuildConfig} config
+ * @returns {Promise<import('../runtime/client/config.js').IScout9ProjectBuildConfig>}
+ */
+export async function syncData(config) {
+  if (!process.env.SCOUT9_API_KEY) {
+    throw new Error('Missing required environment variable "SCOUT9_API_KEY"');
+  }
+  const result = await platformApi(`https://scout9.com/api/b/platform/sync`).then((res) => {
+    if (res.status !== 200) {
+      throw new Error(`Server responded with ${res.status}: ${res.statusText}`);
+    }
+    return res.json();
+  })
+    .catch((err) => {
+      err.message = `Error fetching entities and agents: ${err.message}`;
+      throw err;
+    });
+
+  const { agents, entities, organization, initialContext } = result;
+
+  // Merge
+  config.agents = agents.reduce((accumulator, agent) => {
+    // Check if agent already exists
+    const existingAgentIndex = accumulator.findIndex(a => a.id === agent.id);
+    if (existingAgentIndex === -1) {
+      accumulator.push(agent);
+    } else {
+      // Merge agent
+      accumulator[existingAgentIndex] = {
+        ...accumulator[existingAgentIndex],
+        ...agent
+      };
+    }
+    return accumulator;
+  }, config.agents);
+
+  // Remove agents that are not on the server
+  config.agents = config.agents.filter(agent => agents.find(a => a.id === agent.id));
+
+  // Merge entities
+  config.entities = entities.reduce((accumulator, entity) => {
+    // Check if agent already exists
+    const existingEntityIndex = accumulator.findIndex(a => a.id === entity.id);
+    if (existingEntityIndex === -1) {
+      accumulator.push(entity);
+    } else {
+      // Merge agent
+      accumulator[existingEntityIndex] = {
+        ...accumulator[existingEntityIndex],
+        ...entity
+      };
+    }
+    return accumulator;
+  }, config.entities);
+
+  // Remove entities that are not on the server
+  config.entities = config.entities.filter(entity => entities.find(a => a.id === entity.id));
+  config.organization = {
+    ...(config?.organization || {}),
+    ...(organization || {})
+  };
+  config.initialContext = [
+    ...(Array.isArray(config?.initialContext)  ? config.initialContext : []),
+    ...(Array.isArray(initialContext) ? initialContext : [])
+  ];
+
+  return config;
+}

package/src/core/templates/Dockerfile

@@ -0,0 +1,22 @@
+# Use Node.js version 16
+FROM node:18-slim
+
+# Create and change to the app directory.
+WORKDIR /usr/src/app
+
+# Copy application dependency manifests to the container image.
+# A wildcard is used to ensure both package.json AND package-lock.json are copied.
+# Copying this separately prevents re-running npm install on every code change.
+COPY package*.json ./
+
+# Install production dependencies.
+RUN npm install --only=production
+
+# Copy local code to the container image.
+COPY . ./
+
+# Expose port for Cloud Run
+EXPOSE 8080
+
+# Run the web service on container startup.
+CMD [ "npm", "start" ]

package/src/core/templates/app.js

@@ -0,0 +1,453 @@
+import polka from 'polka';
+import sirv from 'sirv';
+import compression from 'compression';
+import bodyParser from 'body-parser';
+import colors from 'kleur';
+import { config as dotenv } from 'dotenv';
+import { Configuration, Scout9Api } from '@scout9/admin';
+import { EventResponse } from '@scout9/app';
+import path from 'node:path';
+import fs from 'node:fs';
+import https from 'node:https';
+import { fileURLToPath, pathToFileURL } from 'node:url';
+import projectApp from './src/app.js';
+import config from './config.js';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+
+const dev = process.env.DEV_MODE === 'true' || process.env.TEST_MODE === 'true';
+
+class ServerCache {
+  constructor(filePath = path.resolve(__dirname, './server.cache.json')) {
+    this.filePath = filePath;
+    this._load();
+  }
+
+  isTested() {
+    return this._cache.tested === true;
+  }
+
+  setTested(value = true) {
+    this._cache.tested = value;
+    this._save();
+  }
+
+  reset(override = {tested: false}) {
+    this._save(override);
+    this._cache = override;
+  }
+
+  _load() {
+    try {
+      this._cache = JSON.parse(fs.readFileSync(this.filePath, 'utf8'));
+      return this._cache;
+    } catch (e) {
+      if (e.code === 'ENOENT') {
+        this._save();
+      } else {
+        throw e;
+      }
+    }
+  }
+
+  _save(override) {
+    fs.writeFileSync(this.filePath, JSON.stringify(override || this._cache || {tested: false}));
+  }
+}
+
+
+// Ensure .env config is set (specifically SCOUT9_API_KEY)
+const configFilePath = path.resolve(process.cwd(), './.env');
+dotenv({path: configFilePath});
+
+const configuration = new Configuration({
+  apiKey: process.env.SCOUT9_API_KEY || ''
+});
+const scout9 = new Scout9Api(configuration);
+const cache = new ServerCache();
+cache.reset();
+
+
+const handleError = (e, res = undefined) => {
+  let name = e?.name || 'Runtime Error';
+  let message = e?.message || 'Unknown error';
+  let code = typeof e?.code === 'number'
+    ? e?.code
+    : typeof e?.status === 'number'
+      ? e?.status
+      : 500;
+  if ('response' in e) {
+    const response = e.response;
+    if (response?.status) {
+      code = response.status;
+    }
+    if (response?.statusText) {
+      name = response.statusText;
+    }
+    if (response?.data) {
+      message = response.data;
+    } else if (response?.body) {
+      message = response.body;
+    }
+  }
+  console.log(colors.red(`${colors.bold(`${code} Error`)}: ${message}`));
+  if (res) {
+    res.writeHead(code, {'Content-Type': 'application/json'});
+    res.end(JSON.stringify({
+      name,
+      error: message,
+      code
+    }));
+  }
+};
+
+const makeRequest = async (options, maxRedirects = 10) => {
+  return new Promise((resolve, reject) => {
+
+    if (maxRedirects < 0) {
+      reject(new Error('Too many redirects'));
+      return;
+    }
+
+    const req = https.request(options, (res) => {
+      console.log(`STATUS: ${res.statusCode}`);
+      if (res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) {
+        // Handle redirect
+        console.log(`Redirecting to ${res.headers.location}`);
+        const newUrl = new URL(res.headers.location);
+        const newOptions = {
+          hostname: newUrl.hostname,
+          port: newUrl.port || 443,
+          path: newUrl.pathname,
+          method: 'GET', // Usually redirects are GET, adjust if necessary
+          headers: options.headers // Reuse original headers
+          // Add any other necessary options here
+        };
+        // Recursive call to handle redirect
+        resolve(makeRequest(newOptions, maxRedirects - 1));
+      } else {
+        let data = '';
+        res.setEncoding('utf8');
+        res.on('data', (chunk) => data += chunk);
+        res.on('end', () => {
+          try {
+            resolve(JSON.parse(data));
+          } catch (e) {
+            reject(e);
+          }
+        });
+      }
+    });
+
+    req.on('error', (e) => {
+      reject(e);
+    });
+
+    req.end();
+
+  });
+};
+
+
+const app = polka();
+
+app.use(bodyParser.json());
+
+if (dev) {
+  app.use(compression());
+  app.use(sirv(path.resolve(__dirname, 'public'), {dev: true}));
+}
+
+// Root application POST endpoint will run the scout9 app
+app.post(dev ? '/dev/workflow' : '/', async (req, res) => {
+  try {
+    // @TODO use zod to check if req.body is a valid event object
+    const response = await projectApp(req.body);
+    res.writeHead(200, {'Content-Type': 'application/json'});
+    res.end(JSON.stringify(response));
+  } catch (e) {
+    handleError(e, res);
+  }
+});
+
+function isSurroundedByBrackets(str) {
+  return /^\[.*\]$/.test(str);
+}
+
+function resolveEntity(entity, method) {
+  const entityField = config.entities.find(e => e.entity === entity);
+  if (!entityField) {
+    console.error(`Invalid entity: "${entity}" not found within [${config.entities.map(e => e.entity).join(', ')}]`);
+    throw new Error(`Invalid entity: not found`);
+  }
+  const {api, entities} = entityField;
+  if (!api && !api[method]) {
+    throw new Error(`Invalid entity: no API`);
+  }
+  if (!entities) {
+    throw new Error(`Invalid entity: no path`);
+  }
+  return entityField;
+}
+
+async function resolveEntityApi(entity, method) {
+  const paramEntity = isSurroundedByBrackets(entity);
+  if (method === 'GET' && !paramEntity) {
+    method = 'QUERY';
+  }
+  const methods = ['GET', 'UPDATE', 'QUERY', 'PUT', 'PATCH', 'DELETE'];
+  if (!methods.includes(method)) {
+    throw new Error(`Invalid method: ${method}`);
+  }
+  const {api, entities} = resolveEntity(entity, method);
+  const mod = await import(pathToFileURL(path.resolve(__dirname, `./src/entities/${entities.join('/')}/api.js`)).href)
+    .catch((e) => {
+      switch (e.code) {
+        case 'ERR_MODULE_NOT_FOUND':
+        case 'MODULE_NOT_FOUND':
+          console.error(e);
+          throw new Error(`Invalid entity: no API method`);
+        default:
+          console.error(e);
+          throw new Error(`Invalid entity: Internal system error`);
+      }
+    });
+  if (mod[method]) {
+    return mod[method];
+  }
+
+  if (method === 'QUERY' && mod['GET']) {
+    return mod['GET'];
+  }
+
+  throw new Error(`Invalid entity: no API method`);
+
+}
+
+function extractParamsFromPath(path) {
+  const segments = path.split('/').filter(Boolean); // Split and remove empty segments
+  let params = {};
+  const dataStructure = config.entities;
+
+  // Assuming the structure starts with "/entity/"
+  segments.shift(); // remove the "entity" segment
+
+  let lastEntity;
+  let lastSegment;
+  segments.forEach(segment => {
+    const isEntity = dataStructure.some(d => d.entity === segment || d.entity === `[${segment}]`);
+    if (isEntity) {
+      lastEntity = segment;
+      lastSegment = segment;
+    } else if (lastEntity) {
+      const entityDefinition = dataStructure.find(d => {
+        const index = d.entities.indexOf(lastEntity);
+        return index > -1 && index === (d.entities.length - 2);
+      });
+      if (entityDefinition) {
+        const paramName = entityDefinition.entity.replace(/[\[\]]/g, ''); // Remove brackets to get the param name
+        params[paramName] = segment;
+        lastSegment = entityDefinition.entity;
+      }
+      lastEntity = null; // Reset for next potential entity-instance pair
+    }
+  });
+
+  return {params, lastEntity, lastSegment};
+}
+
+async function runEntityApi(req, res) {
+  try {
+    // polka doesn't support wildcards
+    const {params, lastSegment} = extractParamsFromPath(req.url);
+    const api = await resolveEntityApi(lastSegment, req.method.toUpperCase());
+    const response = await api({
+      params,
+      searchParams: req?.query || {}, body: req?.body || undefined,
+      id: params.id
+    });
+    if (response instanceof EventResponse && !!response.body) {
+      res.writeHead(response.status || 200, {'Content-Type': 'application/json'});
+      res.end(JSON.stringify(response.body));
+    } else {
+      throw new Error(`Invalid response: not an EventResponse`);
+    }
+  } catch (e) {
+    console.error(e);
+    res.writeHead(500, {'Content-Type': 'application/json'});
+    res.end(JSON.stringify({error: e.message}));
+  }
+}
+
+app.get('/entity/:entity', runEntityApi);
+app.put('/entity/:entity', runEntityApi);
+app.patch('/entity/:entity', runEntityApi);
+app.post('/entity/:entity', runEntityApi);
+app.delete('/entity/:entity', runEntityApi);
+app.get('/entity/:entity/*', runEntityApi);
+app.put('/entity/:entity/*', runEntityApi);
+app.patch('/entity/:entity/*', runEntityApi);
+app.post('/entity/:entity/*', runEntityApi);
+app.delete('/entity/:entity/*', runEntityApi);
+
+// For local development: parse a message
+if (dev) {
+
+  app.get('/dev/config', async (req, res, next) => {
+
+    // Retrieve auth token
+    const {token, id} = await makeRequest({
+      hostname: 'us-central1-jumpstart.cloudfunctions.net',
+      port: 443,
+      path: '/v1-utils-platform-token',
+      method: 'GET',
+      headers: {
+        'Authorization': 'Bearer ' + process.env.SCOUT9_API_KEY
+      }
+    });
+    res.writeHead(200, {'Content-Type': 'application/json'});
+    res.end(JSON.stringify({token, id, ...config}));
+    try {
+      if (!cache.isTested()) {
+        const testableEntities = config.entities.filter(e => e?.definitions?.length > 0 || e?.training?.length > 0);
+        if (dev && testableEntities.length > 0) {
+          console.log(`${colors.grey(`${colors.cyan('>')} Testing ${colors.bold(colors.white(testableEntities.length))} Entities...`)}`);
+          const _res = await scout9.parse({
+            message: 'Dummy message to parse',
+            language: 'en',
+            entities: testableEntities
+          });
+          cache.setTested();
+          console.log(`\t${colors.green(`+ ${testableEntities.length} Entities passed`)}`);
+        }
+      }
+    } catch (e) {
+      console.error(e);
+      handleError(e);
+    }
+  });
+
+  app.post('/dev/parse', async (req, res, next) => {
+    try {
+      // req.body: {message: string}
+      const {message, language} = req.body;
+      if (typeof message !== 'string') {
+        throw new Error('Invalid message - expected to be a string');
+      }
+      console.log(`${colors.grey(`${colors.cyan('>')} Parsing "${colors.bold(colors.white(message))}`)}"`);
+      const payload = await scout9.parse({
+        message,
+        language: 'en',
+        entities: config.entities
+      }).then((_res => _res.data));
+      let fields = '';
+      for (const [key, value] of Object.entries(payload.context)) {
+        fields += `\n\t\t${colors.bold(colors.white(key))}: ${colors.grey(JSON.stringify(value))}`;
+      }
+      console.log(`\tParsed in ${payload.ms}ms:${colors.grey(`${fields}`)}`);
+      res.writeHead(200, {'Content-Type': 'application/json'});
+      res.end(JSON.stringify(payload));
+    } catch (e) {
+      handleError(e, res);
+    }
+  });
+
+  app.post('/dev/forward', async (req, res, next) => {
+    try {
+      // req.body: {message: string}
+      const {convo} = req.body;
+      console.log(`${colors.grey(`${colors.cyan('>')} Forwarding...`)}`);
+      const payload = await scout9.forward({convo}).then((_res => _res.data));
+      console.log(`\tForwarded in ${payload?.ms}ms`);
+      res.writeHead(200, {'Content-Type': 'application/json'});
+      res.end(JSON.stringify(payload));
+    } catch (e) {
+      handleError(e, res);
+    }
+  });
+
+  app.post('/dev/generate', async (req, res, next) => {
+    try {
+      // req.body: {conversation: {}, messages: []}
+      const {messages, persona: personaId} = req.body;
+      if (typeof messages !== 'object' || !Array.isArray(messages)) {
+        throw new Error('Invalid messages array - expected to be an array of objects');
+      }
+      if (typeof personaId !== 'string') {
+        throw new Error('Invalid persona - expected to be a string');
+      }
+      const persona = (config.persona || config.agents).find(p => p.id === personaId);
+      if (!persona) {
+        throw new Error(`Could not find persona with id: ${personaId}, ensure your project is sync'd by running "scout9 sync"`);
+      }
+      console.log(`${colors.grey(`${colors.cyan('>')} Generating ${colors.bold(colors.white(persona.firstName))}'s`)} ${colors.bold(
+        colors.red(colors.bgBlack('auto-reply')))}`);
+      const payload = await scout9.generate({
+        messages,
+        persona,
+        llm: config.llm,
+        pmt: config.pmt
+      }).then((_res => _res.data));
+      console.log(`\t${colors.grey(`Response: ${colors.green('"')}${colors.bold(colors.white(payload.message))}`)}${colors.green(
+        '"')} (elapsed ${payload.ms}ms)`);
+      res.writeHead(200, {'Content-Type': 'application/json'});
+      res.end(JSON.stringify(payload));
+    } catch (e) {
+      handleError(e, res);
+    }
+  });
+}
+
+
+app.listen(process.env.PORT || 8080, err => {
+  if (err) throw err;
+
+  const art_scout9 = `
+
+
+ ________  ________  ________  ___  ___  _________  ________     
+|\\   ____\\|\\   ____\\|\\   __  \\|\\  \\|\\  \\|\\___   ___\\\\  ___  \\    
+\\ \\  \\___|\\ \\  \\___|\\ \\  \\|\\  \\ \\  \\\\\\  \\|___ \\  \\_\\ \\____   \\   
+ \\ \\_____  \\ \\  \\    \\ \\  \\\\\\  \\ \\  \\\\\\  \\   \\ \\  \\ \\|____|\\  \\  
+  \\|____|\\  \\ \\  \\____\\ \\  \\\\\\  \\ \\  \\\\\\  \\   \\ \\  \\    __\\_\\  \\ 
+    ____\\_\\  \\ \\_______\\ \\_______\\ \\_______\\   \\ \\__\\  |\\_______\\
+   |\\_________\\|_______|\\|_______|\\|_______|    \\|__|  \\|_______|
+   \\|_________|                                                  
+`;
+  const art_auto_reply = `    ___         __           ____             __         __   
+   /   | __  __/ /_____     / __ \\___  ____  / /_  __   / /   
+  / /| |/ / / / __/ __ \\   / /_/ / _ \\/ __ \\/ / / / /  / /    
+ / ___ / /_/ / /_/ /_/ /  / _, _/  __/ /_/ / / /_/ /  /_/     
+/_/  |_\\__,_/\\__/\\____/  /_/ |_|\\___/ .___/_/\\__, /  (_)      
+                                   /_/      /____/            
+  
+`;
+  const protocol = process.env.PROTOCOL || 'http';
+  const host = process.env.HOST || 'localhost';
+  const port = process.env.PORT || 8080;
+  const fullUrl = `${protocol}://${host}:${port}`;
+  if (dev) {
+    console.log(colors.bold(colors.green(art_scout9)));
+    console.log(colors.bold(colors.cyan(art_auto_reply)));
+    console.log(`${colors.grey(`${colors.cyan('>')} Running ${colors.bold(colors.white('Scout9'))}`)} ${colors.bold(
+      colors.red(colors.bgBlack('auto-reply')))} ${colors.grey('dev environment on')} ${fullUrl}`);
+  } else {
+    console.log(`Running Scout9 auto-reply app on ${fullUrl}`);
+  }
+  // Run checks
+  if (!fs.existsSync(configFilePath)) {
+    console.log(colors.red('Missing .env file, your auto reply application may not work without it.'));
+  }
+
+  if (dev && !process.env.SCOUT9_API_KEY) {
+    console.log(colors.red(
+      'Missing SCOUT9_API_KEY environment variable, your auto reply application may not work without it.'));
+  }
+
+  if (process.env.SCOUT9_API_KEY === '<insert-scout9-api-key>') {
+    console.log(`${colors.red('SCOUT9_API_KEY has not been set in your .env file.')} ${colors.grey(
+      'You can find your API key in the Scout9 dashboard.')} ${colors.bold(colors.cyan('https://scout9.com'))}`);
+  }
+
+});

package/src/core/templates/project-files.js

@@ -0,0 +1,36 @@
+import fs from 'node:fs/promises';
+import { globSync } from 'glob';
+
+
+const resolveFilePath = () => {
+  const paths = globSync(`${src}/entities/agents/{index,config}.{ts,js}`, {cwd, absolute: true});
+  if (paths.length === 0) {
+    throw new Error(`Missing required agents entity file, rerun "scout9 sync" to fix`);
+  }
+  if (paths.length > 1) {
+    throw new Error(`Multiple agents entity files found, rerun "scout9 sync" to fix`);
+  }
+}
+
+export const entities = {
+  /**
+   * Generates ./src/entities/
+   * @param {import('../../runtime/client/config.js').IScout9ProjectBuildConfig} config
+   * @param {string} cwd
+   * @param {string} src
+   * @returns {Promise<void>}
+   */
+
+  agents: async function (config, cwd = process.cwd(), src = './src') {
+    const content = `
+/**
+ * Required core entity type: Agents represents you and your team
+ * @returns {Array<Agent>}
+ */
+export default function Agents() {
+  return ${JSON.stringify(config.agents, null, 2)};
+}
+`;
+    await fs.writeFile(filePath, );
+  }
+}

package/src/core/templates/template-package.json

@@ -0,0 +1,13 @@
+{
+  "name": "templates",
+  "version": "1.0.0",
+  "description": "Used to ensure template app.js gets the correct dependencies.",
+  "dependencies": {
+    "body-parser": "latest",
+    "compression": "latest",
+    "dotenv": "latest",
+    "kleur": "latest",
+    "polka": "latest",
+    "sirv": "latest"
+  }
+}

package/src/exports.js

@@ -2,35 +2,39 @@ import { Scout9Platform } from './platform.js';
 import { EventResponse } from './runtime/index.js';
 
 export { EventResponse } from './runtime/index.js';
-// export * from './types';
+
 export * from './testing-tools/index.js';
+export * from './runtime/client/index.js';
 
 /**
- * @param event
- * @param cwd
- * @param folder
- * @returns {Promise<*|undefined>}
+ * @param {import('./runtime/client/workflow.js').IWorkflowEvent} event - every workflow receives an event object
+ * @param {Object} options
+ * @param {string} [options.cwd=process.cwd()] - the working directory
+ * @param {string} [options.mode='production'] - the build mode
+ * @param {string} [options.src='./src'] - the source directory
+ * @param {string} options.eventSource - the source of the workflow event
+ * @returns {Promise<import('./runtime/client/workflow.js').IWorkflowResponse>}
  */
-export async function run(
-  event,
-  {cwd = process.cwd(), folder} = {},
-) {
-  return Scout9Platform.run(event, {cwd, folder})
+export async function run(event, options) {
+  return Scout9Platform.run(event, options)
 }
 
-export async function sendEvent(
-  event,
-  {cwd = process.cwd(), folder} = {},
-) {
-  return Scout9Platform.run(event, {cwd, folder})
-}
+/**
+ * @param {import('./runtime/client/workflow.js').IWorkflowEvent} event - every workflow receives an event object
+ * @param {{cwd: string; mode: 'development' | 'production'; src: string}} options - build options
+ * @returns {Promise<import('./runtime/client/workflow.js').IWorkflowResponse>}
+ */
+export const sendEvent = run;
 
 /**
  * @param data {T}
  * @param init {ResponseInit | undefined}
- * @returns {EventResponse<T>}
+ * @returns {import('./runtime/client/api.js').EventResponse<T>}
  */
 export function json(data, init) {
+  if (data instanceof Promise) {
+    throw new Error(`json() does not expect a Promise. Use json(await promise) instead`);
+  }
   // TODO deprecate this in favour of `Response.json` when it's
   // more widely supported
   const body = JSON.stringify(data);