@jaimevalasek/aioson 1.3.0 → 1.5.1

package/src/web.js

@@ -0,0 +1,284 @@
+'use strict';
+
+const DEFAULT_TIMEOUT_MS = 10000;
+const DEFAULT_MAX_HTML_CHARS = 500000;
+const DEFAULT_USER_AGENT = 'AIOSON-Web/1.0 (+https://aioson.dev)';
+
+function ensureValidUrl(input) {
+  try {
+    return new URL(String(input || '').trim());
+  } catch {
+    throw new Error(`Invalid URL: ${input || ''}`);
+  }
+}
+
+function normalizeUrl(input) {
+  const url = input instanceof URL ? new URL(input.toString()) : ensureValidUrl(input);
+  url.hash = '';
+  if ((url.protocol === 'http:' && url.port === '80') || (url.protocol === 'https:' && url.port === '443')) {
+    url.port = '';
+  }
+  const normalized = url.toString();
+  return normalized.endsWith('/') ? normalized.slice(0, -1) : normalized;
+}
+
+function decodeHtmlEntities(text) {
+  return String(text || '')
+    .replace(/&#(\d+);/g, (_, code) => String.fromCodePoint(Number(code)))
+    .replace(/&#x([0-9a-f]+);/gi, (_, code) => String.fromCodePoint(parseInt(code, 16)))
+    .replace(/ /gi, ' ')
+    .replace(/&/gi, '&')
+    .replace(/"/gi, '"')
+    .replace(/'/gi, "'")
+    .replace(/&lt;/gi, '<')
+    .replace(/&gt;/gi, '>');
+}
+
+function stripTags(html) {
+  return decodeHtmlEntities(String(html || '').replace(/<[^>]+>/g, ' '));
+}
+
+function cleanWhitespace(text) {
+  return String(text || '')
+    .replace(/\r/g, '')
+    .replace(/\t/g, ' ')
+    .replace(/[ \u00A0]+\n/g, '\n')
+    .replace(/\n[ \u00A0]+/g, '\n')
+    .replace(/[ \u00A0]{2,}/g, ' ')
+    .replace(/\n{3,}/g, '\n\n')
+    .trim();
+}
+
+function extractTagContent(html, tagName) {
+  const match = String(html || '').match(new RegExp(`<${tagName}[^>]*>([\\s\\S]*?)<\\/${tagName}>`, 'i'));
+  return match ? match[1] : '';
+}
+
+function extractTitle(html) {
+  return cleanWhitespace(stripTags(extractTagContent(html, 'title')));
+}
+
+function extractMetaContent(html, name) {
+  const match = String(html || '').match(new RegExp(`<meta[^>]+(?:name|property)=['\"]${name}['\"][^>]+content=['\"]([^'\"]+)['\"][^>]*>`, 'i'));
+  return match ? decodeHtmlEntities(match[1]).trim() : '';
+}
+
+function extractCanonical(html) {
+  const match = String(html || '').match(/<link[^>]+rel=['\"]canonical['\"][^>]+href=['\"]([^'\"]+)['\"][^>]*>/i);
+  return match ? match[1].trim() : '';
+}
+
+function extractMainHtml(html) {
+  const source = String(html || '');
+  const patterns = [
+    /<main\b[^>]*>([\s\S]*?)<\/main>/i,
+    /<article\b[^>]*>([\s\S]*?)<\/article>/i,
+    /<div\b[^>]*role=['\"]main['\"][^>]*>([\s\S]*?)<\/div>/i,
+    /<body\b[^>]*>([\s\S]*?)<\/body>/i
+  ];
+
+  for (const pattern of patterns) {
+    const match = source.match(pattern);
+    if (match && match[1]) return match[1];
+  }
+  return source;
+}
+
+function htmlToMarkdown(html) {
+  let output = String(html || '');
+  output = output.replace(/<script\b[^>]*>[\s\S]*?<\/script>/gi, ' ');
+  output = output.replace(/<style\b[^>]*>[\s\S]*?<\/style>/gi, ' ');
+  output = output.replace(/<noscript\b[^>]*>[\s\S]*?<\/noscript>/gi, ' ');
+  output = output.replace(/<svg\b[^>]*>[\s\S]*?<\/svg>/gi, ' ');
+  output = output.replace(/<h1\b[^>]*>([\s\S]*?)<\/h1>/gi, '\n# $1\n\n');
+  output = output.replace(/<h2\b[^>]*>([\s\S]*?)<\/h2>/gi, '\n## $1\n\n');
+  output = output.replace(/<h3\b[^>]*>([\s\S]*?)<\/h3>/gi, '\n### $1\n\n');
+  output = output.replace(/<h4\b[^>]*>([\s\S]*?)<\/h4>/gi, '\n#### $1\n\n');
+  output = output.replace(/<h5\b[^>]*>([\s\S]*?)<\/h5>/gi, '\n##### $1\n\n');
+  output = output.replace(/<h6\b[^>]*>([\s\S]*?)<\/h6>/gi, '\n###### $1\n\n');
+  output = output.replace(/<li\b[^>]*>([\s\S]*?)<\/li>/gi, '\n- $1');
+  output = output.replace(/<(p|div|section|article|main|header|footer|aside|blockquote)\b[^>]*>/gi, '\n');
+  output = output.replace(/<\/(p|div|section|article|main|header|footer|aside|blockquote)>/gi, '\n');
+  output = output.replace(/<br\s*\/?>/gi, '\n');
+  output = output.replace(/<a\b[^>]*href=['\"]([^'\"]+)['\"][^>]*>([\s\S]*?)<\/a>/gi, '$2 ($1)');
+  output = stripTags(output);
+  return cleanWhitespace(output);
+}
+
+function htmlToText(html) {
+  return cleanWhitespace(stripTags(String(html || '')
+    .replace(/<script\b[^>]*>[\s\S]*?<\/script>/gi, ' ')
+    .replace(/<style\b[^>]*>[\s\S]*?<\/style>/gi, ' ')
+    .replace(/<noscript\b[^>]*>[\s\S]*?<\/noscript>/gi, ' ')
+    .replace(/<br\s*\/?>/gi, '\n')
+    .replace(/<\/(p|div|section|article|main|header|footer|aside|li|ul|ol|h1|h2|h3|h4|h5|h6)>/gi, '\n')));
+}
+
+function resolveHref(href, pageUrl) {
+  const value = String(href || '').trim();
+  if (!value || value.startsWith('#') || value.startsWith('mailto:') || value.startsWith('tel:') || value.startsWith('javascript:')) {
+    return '';
+  }
+
+  try {
+    return normalizeUrl(new URL(value, pageUrl));
+  } catch {
+    return '';
+  }
+}
+
+function extractLinks(html, pageUrl, options = {}) {
+  const sameOriginOnly = options.sameOriginOnly !== false;
+  const base = ensureValidUrl(pageUrl);
+  const links = new Set();
+  const pattern = /<a\b[^>]*href=['\"]([^'\"]+)['\"][^>]*>/gi;
+  let match;
+
+  while ((match = pattern.exec(String(html || ''))) !== null) {
+    const normalized = resolveHref(match[1], base);
+    if (!normalized) continue;
+    if (sameOriginOnly) {
+      try {
+        if (new URL(normalized).origin !== base.origin) continue;
+      } catch {
+        continue;
+      }
+    }
+    links.add(normalized);
+  }
+
+  return Array.from(links).sort();
+}
+
+async function fetchPage(url, options = {}) {
+  const timeoutMs = Number(options.timeoutMs) > 0 ? Number(options.timeoutMs) : DEFAULT_TIMEOUT_MS;
+  const maxHtmlChars = Number(options.maxHtmlChars) > 0 ? Number(options.maxHtmlChars) : DEFAULT_MAX_HTML_CHARS;
+  const headers = {
+    'user-agent': options.userAgent || DEFAULT_USER_AGENT,
+    'accept': 'text/html,application/xhtml+xml,text/plain;q=0.9,*/*;q=0.8',
+    ...(options.headers || {})
+  };
+
+  const response = await fetch(String(url), {
+    method: 'GET',
+    headers,
+    redirect: 'follow',
+    signal: AbortSignal.timeout(timeoutMs)
+  });
+
+  const finalUrl = normalizeUrl(response.url || url);
+  const contentType = response.headers.get('content-type') || '';
+  const rawText = await response.text();
+  const html = rawText.length > maxHtmlChars ? rawText.slice(0, maxHtmlChars) : rawText;
+
+  return {
+    ok: response.ok,
+    url: finalUrl,
+    statusCode: response.status,
+    contentType,
+    html,
+    truncated: rawText.length > html.length
+  };
+}
+
+async function scrapePage(url, options = {}) {
+  const page = await fetchPage(url, options);
+  const mainHtml = extractMainHtml(page.html);
+  const title = extractTitle(page.html);
+  const description = extractMetaContent(page.html, 'description') || extractMetaContent(page.html, 'og:description');
+  const canonical = extractCanonical(page.html);
+  const links = extractLinks(page.html, page.url, { sameOriginOnly: options.sameOriginOnly !== false });
+  const text = htmlToText(mainHtml);
+  const markdown = htmlToMarkdown(mainHtml);
+
+  return {
+    ok: page.ok,
+    url: page.url,
+    statusCode: page.statusCode,
+    contentType: page.contentType,
+    title,
+    description,
+    canonical,
+    html: mainHtml,
+    text,
+    markdown,
+    links,
+    truncated: page.truncated
+  };
+}
+
+async function mapWebsite(url, options = {}) {
+  const startUrl = normalizeUrl(url);
+  const start = ensureValidUrl(startUrl);
+  const maxDepth = Number(options.maxDepth) >= 0 ? Number(options.maxDepth) : 2;
+  const maxPages = Number(options.maxPages) > 0 ? Number(options.maxPages) : 25;
+  const sameOriginOnly = options.sameOriginOnly !== false;
+
+  const queue = [{ url: startUrl, depth: 0 }];
+  const visited = new Set();
+  const pages = [];
+
+  while (queue.length > 0 && visited.size < maxPages) {
+    const current = queue.shift();
+    if (!current || visited.has(current.url)) continue;
+    visited.add(current.url);
+
+    try {
+      const page = await scrapePage(current.url, options);
+      const sameOriginLinks = extractLinks(page.html || '', page.url, { sameOriginOnly });
+      pages.push({
+        url: page.url,
+        depth: current.depth,
+        statusCode: page.statusCode,
+        title: page.title,
+        description: page.description,
+        linkCount: sameOriginLinks.length
+      });
+
+      if (current.depth >= maxDepth) continue;
+      for (const link of sameOriginLinks) {
+        if (visited.has(link)) continue;
+        if (sameOriginOnly) {
+          try {
+            if (new URL(link).origin != start.origin) continue;
+          } catch {
+            continue;
+          }
+        }
+        queue.push({ url: link, depth: current.depth + 1 });
+      }
+    } catch (error) {
+      pages.push({
+        url: current.url,
+        depth: current.depth,
+        statusCode: 0,
+        title: '',
+        description: '',
+        linkCount: 0,
+        error: error.message
+      });
+    }
+  }
+
+  return {
+    startUrl,
+    maxDepth,
+    maxPages,
+    pageCount: pages.length,
+    urls: pages.map((item) => item.url),
+    pages
+  };
+}
+
+module.exports = {
+  DEFAULT_TIMEOUT_MS,
+  DEFAULT_MAX_HTML_CHARS,
+  DEFAULT_USER_AGENT,
+  normalizeUrl,
+  extractLinks,
+  htmlToMarkdown,
+  htmlToText,
+  fetchPage,
+  scrapePage,
+  mapWebsite
+};

package/src/worker-runner.js

@@ -0,0 +1,339 @@
+'use strict';
+
+const fs = require('node:fs/promises');
+const path = require('node:path');
+const { spawn } = require('node:child_process');
+
+const SQUADS_DIR = path.join('.aioson', 'squads');
+const DEFAULT_TIMEOUT = 30000;
+const DEFAULT_RETRY_ATTEMPTS = 3;
+const BACKOFF_BASE = [1000, 3000, 8000];
+
+async function pathExists(targetPath) {
+  try {
+    await fs.access(targetPath);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+async function loadWorkerConfig(projectDir, squadSlug, workerSlug) {
+  const configPath = path.join(projectDir, SQUADS_DIR, squadSlug, 'workers', workerSlug, 'worker.json');
+  if (!(await pathExists(configPath))) return null;
+  const raw = await fs.readFile(configPath, 'utf8');
+  return JSON.parse(raw);
+}
+
+async function listWorkers(projectDir, squadSlug) {
+  const workersDir = path.join(projectDir, SQUADS_DIR, squadSlug, 'workers');
+  let entries;
+  try {
+    entries = await fs.readdir(workersDir, { withFileTypes: true });
+  } catch {
+    return [];
+  }
+  const workers = [];
+  for (const entry of entries) {
+    if (!entry.isDirectory()) continue;
+    const config = await loadWorkerConfig(projectDir, squadSlug, entry.name);
+    if (config) {
+      workers.push({ slug: entry.name, ...config });
+    }
+  }
+  return workers;
+}
+
+function resolveScript(projectDir, squadSlug, workerSlug) {
+  const base = path.join(projectDir, SQUADS_DIR, squadSlug, 'workers', workerSlug);
+  return {
+    js: path.join(base, 'run.js'),
+    py: path.join(base, 'run.py')
+  };
+}
+
+function validateInputs(inputPayload, inputSchema) {
+  if (!inputSchema) return { valid: true, errors: [] };
+  const errors = [];
+  for (const [key, spec] of Object.entries(inputSchema)) {
+    if (spec.required && (inputPayload[key] === undefined || inputPayload[key] === null)) {
+      errors.push(`Missing required input: ${key}`);
+    }
+  }
+  return { valid: errors.length === 0, errors };
+}
+
+function spawnWorker(scriptPath, inputPayload, env, timeoutMs) {
+  return new Promise((resolve, reject) => {
+    const ext = path.extname(scriptPath);
+    const cmd = ext === '.py' ? 'python3' : 'node';
+    const args = [scriptPath, JSON.stringify(inputPayload)];
+    const child = spawn(cmd, args, {
+      env: { ...process.env, ...env },
+      stdio: ['ignore', 'pipe', 'pipe'],
+      timeout: timeoutMs
+    });
+
+    let stdout = '';
+    let stderr = '';
+    child.stdout.on('data', (chunk) => { stdout += chunk; });
+    child.stderr.on('data', (chunk) => { stderr += chunk; });
+
+    child.on('close', (code) => {
+      if (code === 0) {
+        let output;
+        try {
+          output = JSON.parse(stdout);
+        } catch {
+          output = { raw: stdout.trim() };
+        }
+        resolve({ ok: true, output, stderr: stderr.trim() || null });
+      } else {
+        let errorDetail;
+        try {
+          errorDetail = JSON.parse(stderr);
+        } catch {
+          errorDetail = { message: stderr.trim() || `Process exited with code ${code}` };
+        }
+        resolve({ ok: false, error: errorDetail.error || errorDetail.message || String(errorDetail), code });
+      }
+    });
+
+    child.on('error', (err) => {
+      if (err.code === 'ETIMEDOUT' || err.killed) {
+        resolve({ ok: false, error: `Worker timed out after ${timeoutMs}ms`, code: -1 });
+      } else {
+        reject(err);
+      }
+    });
+  });
+}
+
+function resolveEnvVars(envKeys) {
+  const resolved = {};
+  for (const key of (envKeys || [])) {
+    if (process.env[key]) {
+      resolved[key] = process.env[key];
+    }
+  }
+  return resolved;
+}
+
+function sleep(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+async function runWorker(projectDir, squadSlug, workerSlug, inputPayload, options = {}) {
+  const config = await loadWorkerConfig(projectDir, squadSlug, workerSlug);
+  if (!config) {
+    return { ok: false, error: `Worker config not found: ${workerSlug}`, attempts: 0 };
+  }
+
+  // Validate inputs
+  const validation = validateInputs(inputPayload || {}, config.inputs);
+  if (!validation.valid) {
+    return { ok: false, error: `Input validation failed: ${validation.errors.join(', ')}`, attempts: 0 };
+  }
+
+  // Resolve script path
+  const scripts = resolveScript(projectDir, squadSlug, workerSlug);
+  let scriptPath;
+  if (await pathExists(scripts.js)) {
+    scriptPath = scripts.js;
+  } else if (await pathExists(scripts.py)) {
+    scriptPath = scripts.py;
+  } else {
+    return { ok: false, error: `No run script found for worker: ${workerSlug}`, attempts: 0 };
+  }
+
+  // Resolve env vars
+  const env = resolveEnvVars(config.env);
+
+  // Resolve MCP env vars if worker declares uses_mcp
+  if (config.uses_mcp && config.uses_mcp.length > 0) {
+    try {
+      const { listIntegrations, buildWorkerMcpEnv } = require('./mcp-connectors/registry');
+      const integrations = await listIntegrations(projectDir, squadSlug);
+      const mcpEnv = buildWorkerMcpEnv(projectDir, squadSlug, config.uses_mcp, integrations);
+      Object.assign(env, mcpEnv);
+    } catch { /* MCP resolution optional — fail silently */ }
+  }
+
+  // Timeout and retry config
+  const timeoutMs = config.timeout_ms || DEFAULT_TIMEOUT;
+  const maxAttempts = (config.retry && config.retry.attempts) || (options.noRetry ? 1 : DEFAULT_RETRY_ATTEMPTS);
+  const triggerType = options.triggerType || 'manual';
+
+  let lastResult;
+  const startTime = Date.now();
+
+  for (let attempt = 1; attempt <= maxAttempts; attempt++) {
+    lastResult = await spawnWorker(scriptPath, inputPayload || {}, env, timeoutMs);
+    lastResult.attempt = attempt;
+    lastResult.triggerType = triggerType;
+
+    if (lastResult.ok) {
+      lastResult.durationMs = Date.now() - startTime;
+      return lastResult;
+    }
+
+    // Backoff before retry
+    if (attempt < maxAttempts) {
+      const delay = BACKOFF_BASE[attempt - 1] || BACKOFF_BASE[BACKOFF_BASE.length - 1];
+      await sleep(delay);
+    }
+  }
+
+  lastResult.durationMs = Date.now() - startTime;
+  return lastResult;
+}
+
+// --- Scaffold generation ---
+
+function generateWorkerJson(slug, name, triggerType, inputFields, outputFields, envVars) {
+  const trigger = { type: triggerType || 'manual' };
+  if (triggerType === 'event') {
+    trigger.source = 'content_item_created';
+    trigger.filter = {};
+  }
+  if (triggerType === 'scheduled') {
+    trigger.cron = '0 8 * * *';
+  }
+
+  const inputs = {};
+  for (const field of (inputFields || [])) {
+    inputs[field] = { type: 'string', required: true };
+  }
+
+  const outputs = {};
+  for (const field of (outputFields || [])) {
+    outputs[field] = { type: 'string' };
+  }
+
+  return {
+    slug,
+    name: name || slug,
+    type: triggerType || 'manual',
+    trigger,
+    inputs,
+    outputs,
+    env: envVars || [],
+    timeout_ms: 30000,
+    retry: { attempts: 3, backoff: 'exponential' }
+  };
+}
+
+function generateRunJs(slug, config) {
+  const inputParsing = Object.keys(config.inputs || {}).length > 0
+    ? Object.keys(config.inputs).map(k => `  // input.${k}`).join('\n')
+    : '  // No inputs defined';
+
+  const outputReturn = Object.keys(config.outputs || {}).length > 0
+    ? '    ' + Object.keys(config.outputs).map(k => `${k}: null`).join(',\n    ')
+    : '    result: "ok"';
+
+  return `#!/usr/bin/env node
+'use strict';
+
+// Worker: ${slug}
+// Trigger: ${config.type || 'manual'}
+// Generated by aioson squad:create
+
+const input = JSON.parse(process.argv[2] || '{}');
+
+async function execute(input) {
+  // Available inputs:
+${inputParsing}
+
+  // TODO: Implement worker logic here
+  // Examples:
+  //   - Send WhatsApp message via Business API
+  //   - Call external REST API
+  //   - Calculate metrics and save to squad_metrics
+  //   - Move a lead in CRM
+  //   - Send email via SMTP
+
+  return {
+${outputReturn}
+  };
+}
+
+execute(input)
+  .then(result => {
+    process.stdout.write(JSON.stringify(result));
+    process.exit(0);
+  })
+  .catch(err => {
+    process.stderr.write(JSON.stringify({ error: err.message }));
+    process.exit(1);
+  });
+`;
+}
+
+function generateWorkerReadme(slug, config) {
+  const envSection = (config.env || []).length > 0
+    ? config.env.map(e => `- \`${e}\``).join('\n')
+    : '- None';
+
+  return `# Worker: ${config.name || slug}
+
+## Trigger
+- **Type**: ${config.type || 'manual'}
+${config.trigger && config.trigger.cron ? `- **Cron**: \`${config.trigger.cron}\`` : ''}
+${config.trigger && config.trigger.source ? `- **Source**: ${config.trigger.source}` : ''}
+
+## Inputs
+${Object.entries(config.inputs || {}).map(([k, v]) => `- \`${k}\` (${v.type})${v.required ? ' *required*' : ''}`).join('\n') || '- None'}
+
+## Outputs
+${Object.entries(config.outputs || {}).map(([k, v]) => `- \`${k}\` (${v.type})`).join('\n') || '- None'}
+
+## Environment Variables
+${envSection}
+
+## Usage
+
+\`\`\`bash
+# Run manually
+aioson squad:worker --squad=<squad-slug> --worker=${slug} --input='{"key": "value"}'
+
+# Test with dry run
+aioson squad:worker --squad=<squad-slug> --worker=${slug} --sub=test
+
+# View execution logs
+aioson squad:worker --squad=<squad-slug> --sub=logs
+\`\`\`
+`;
+}
+
+async function scaffoldWorker(projectDir, squadSlug, workerSlug, options = {}) {
+  const workerDir = path.join(projectDir, SQUADS_DIR, squadSlug, 'workers', workerSlug);
+  await fs.mkdir(workerDir, { recursive: true });
+
+  const config = generateWorkerJson(
+    workerSlug,
+    options.name,
+    options.triggerType,
+    options.inputs,
+    options.outputs,
+    options.env
+  );
+
+  await fs.writeFile(path.join(workerDir, 'worker.json'), JSON.stringify(config, null, 2));
+  await fs.writeFile(path.join(workerDir, 'run.js'), generateRunJs(workerSlug, config));
+  await fs.writeFile(path.join(workerDir, 'README.md'), generateWorkerReadme(workerSlug, config));
+
+  return { workerDir, config };
+}
+
+module.exports = {
+  loadWorkerConfig,
+  listWorkers,
+  runWorker,
+  scaffoldWorker,
+  generateWorkerJson,
+  generateRunJs,
+  generateWorkerReadme,
+  validateInputs,
+  resolveEnvVars
+};