owletto 1.0.0

package/src/cli.ts

@@ -0,0 +1,438 @@
+#!/usr/bin/env node
+
+/**
+ * Owletto SDK CLI
+ *
+ * Compile connectors and generate manifest.json:
+ *
+ *   owletto-sdk compile connectors/reddit.ts connectors/github.ts
+ *   owletto-sdk compile connectors/*.ts
+ *   owletto-sdk compile connectors/        # discover .ts files in dir
+ *   owletto-sdk compile --outdir dist connectors/*.ts
+ */
+
+import {
+  copyFileSync,
+  existsSync,
+  mkdirSync,
+  readdirSync,
+  readFileSync,
+  statSync,
+  unlinkSync,
+  writeFileSync,
+} from 'node:fs';
+import { basename, dirname, join, resolve } from 'node:path';
+import { pathToFileURL } from 'node:url';
+import { build } from 'esbuild';
+
+interface ManifestConnectorEntry {
+  key: string;
+  file: string;
+  name: string;
+  version: string;
+  description?: string;
+  authSchema?: Record<string, unknown>;
+  feeds?: Record<string, unknown>;
+  actions?: Record<string, unknown>;
+  optionsSchema?: Record<string, unknown>;
+}
+
+interface ManifestInsightEntry {
+  slug: string;
+  file: string;
+  name: string;
+  description?: string;
+  version: number;
+  source_types: {
+    required: string[];
+    recommended: string[];
+  };
+}
+
+interface ReleaseManifest {
+  connectors: ManifestConnectorEntry[];
+  insights: ManifestInsightEntry[];
+}
+
+// Back-compat alias
+type ManifestEntry = ManifestConnectorEntry;
+
+/**
+ * Resolve input paths to concrete entry points.
+ *
+ * Accepts a mix of:
+ *   - .ts files  → compile directly
+ *   - directories → discover .ts files inside
+ */
+function resolveEntryPoints(paths: string[]): Array<{ name: string; entryPoint: string }> {
+  const entries: Array<{ name: string; entryPoint: string }> = [];
+
+  for (const p of paths) {
+    const abs = resolve(p);
+    if (!existsSync(abs)) {
+      console.error(`Not found: ${abs}`);
+      process.exit(1);
+    }
+
+    if (statSync(abs).isFile()) {
+      if (!abs.endsWith('.ts')) {
+        console.error(`Not a TypeScript file: ${abs}`);
+        process.exit(1);
+      }
+      entries.push({ name: basename(abs, '.ts'), entryPoint: abs });
+    } else {
+      // Directory: discover .ts files
+      for (const child of readdirSync(abs, { withFileTypes: true })) {
+        if (!child.isFile() || !child.name.endsWith('.ts')) continue;
+        entries.push({
+          name: child.name.replace(/\.ts$/, ''),
+          entryPoint: join(abs, child.name),
+        });
+      }
+    }
+  }
+
+  return entries;
+}
+
+/**
+ * Extract definition metadata from a compiled bundle by dynamically importing
+ * it and finding the ConnectorRuntime subclass.
+ */
+async function extractDefinitionFromBundle(
+  bundlePath: string
+): Promise<Omit<ManifestEntry, 'file'> | null> {
+  const moduleHref = `${pathToFileURL(bundlePath).href}?ts=${Date.now()}`;
+  const mod = (await import(moduleHref)) as Record<string, unknown>;
+
+  // Find ConnectorRuntime subclass: class with .sync() + .execute()
+  const candidates = Object.values(mod);
+  const RuntimeClass = candidates.find(
+    (candidate) =>
+      typeof candidate === 'function' &&
+      !!(candidate as { prototype?: { sync?: unknown; execute?: unknown } }).prototype?.sync &&
+      !!(candidate as { prototype?: { sync?: unknown; execute?: unknown } }).prototype?.execute
+  ) as (new () => { definition: unknown }) | undefined;
+
+  if (!RuntimeClass) return null;
+
+  const instance = new RuntimeClass();
+  const rawDef = instance.definition as Record<string, unknown> | undefined;
+  if (!rawDef) return null;
+
+  const key = typeof rawDef.key === 'string' ? rawDef.key : null;
+  const name = typeof rawDef.name === 'string' ? rawDef.name : null;
+  const version = typeof rawDef.version === 'string' ? rawDef.version : null;
+  if (!key || !name || !version) {
+    throw new Error('Connector definition must include key, name, and version.');
+  }
+
+  return {
+    key,
+    name,
+    version,
+    description: typeof rawDef.description === 'string' ? rawDef.description : undefined,
+    authSchema:
+      rawDef.authSchema && typeof rawDef.authSchema === 'object'
+        ? (rawDef.authSchema as Record<string, unknown>)
+        : undefined,
+    feeds:
+      rawDef.feeds && typeof rawDef.feeds === 'object'
+        ? (rawDef.feeds as Record<string, unknown>)
+        : undefined,
+    actions:
+      rawDef.actions && typeof rawDef.actions === 'object'
+        ? (rawDef.actions as Record<string, unknown>)
+        : undefined,
+    optionsSchema:
+      rawDef.optionsSchema && typeof rawDef.optionsSchema === 'object'
+        ? (rawDef.optionsSchema as Record<string, unknown>)
+        : undefined,
+  };
+}
+
+/**
+ * esbuild plugin to resolve `npm:package@version` specifiers.
+ * Strips the `npm:` prefix and version, leaving just the bare package name
+ * so esbuild resolves it from node_modules.
+ */
+function npmSpecifierPlugin(): import('esbuild').Plugin {
+  return {
+    name: 'npm-specifier',
+    setup(build) {
+      build.onResolve({ filter: /^npm:/ }, (args) => {
+        const specifier = args.path.slice(4); // strip "npm:"
+        const isScoped = specifier.startsWith('@');
+        const match = isScoped
+          ? specifier.match(/^(@[^/]+\/[^/@]+)(?:@[^/]+)?(\/.*)?$/)
+          : specifier.match(/^([^/@]+)(?:@[^/]+)?(\/.*)?$/);
+        if (!match) return { external: true, path: args.path };
+        const pkg = match[1];
+        const subpath = match[2] ?? '';
+        return { path: `${pkg}${subpath}`, external: true };
+      });
+    },
+  };
+}
+
+/**
+ * esbuild plugin that stubs all external runtime dependencies (npm: specifiers,
+ * playwright) with empty modules. Used to build a lightweight "metadata-only"
+ * bundle that can be imported just to read the ConnectorRuntime definition,
+ * without needing any npm packages installed.
+ */
+function stubRuntimeDepsPlugin(): import('esbuild').Plugin {
+  return {
+    name: 'stub-runtime-deps',
+    setup(build) {
+      // Stub npm: specifiers
+      build.onResolve({ filter: /^npm:/ }, (args) => ({
+        path: args.path,
+        namespace: 'stub',
+      }));
+      // Stub playwright
+      build.onResolve({ filter: /^playwright/ }, (args) => ({
+        path: args.path,
+        namespace: 'stub',
+      }));
+      // Return a CJS module whose exports are a callable/constructable proxy.
+      // __esModule must be falsy so esbuild's __toESM sets target.default = mod,
+      // making default imports resolve to the proxy (which is constructable).
+      build.onLoad({ filter: /.*/, namespace: 'stub' }, () => ({
+        contents: `const stub = new Proxy(function(){}, { get: (_, k) => k === '__esModule' ? undefined : stub, apply: () => stub, construct: () => stub }); module.exports = stub;`,
+        loader: 'js',
+      }));
+    },
+  };
+}
+
+/**
+ * Build a temporary "metadata-only" bundle with all runtime deps stubbed,
+ * extract the definition, then clean up.
+ */
+async function extractDefinitionViaStubBuild(
+  entryPoint: string,
+  outputDir: string,
+  sdkEntry: string,
+  name: string
+): Promise<Omit<ManifestEntry, 'file'> | null> {
+  const stubFile = join(outputDir, `.${name}.meta.mjs`);
+  try {
+    await build({
+      entryPoints: [entryPoint],
+      bundle: true,
+      format: 'esm',
+      platform: 'node',
+      target: 'node20',
+      outfile: stubFile,
+      alias: { owletto: sdkEntry },
+      banner: {
+        js: `import { createRequire as __createRequire } from 'module'; const require = __createRequire(import.meta.url);`,
+      },
+      plugins: [stubRuntimeDepsPlugin()],
+      minify: false,
+      sourcemap: false,
+      logLevel: 'silent',
+    });
+    return await extractDefinitionFromBundle(stubFile);
+  } finally {
+    try {
+      unlinkSync(stubFile);
+    } catch {}
+  }
+}
+
+async function compile(inputPaths: string[], outdir: string) {
+  const entries = resolveEntryPoints(inputPaths);
+  if (entries.length === 0) {
+    console.error('No connector source files found.');
+    process.exit(1);
+  }
+
+  const outputDir = resolve(outdir);
+  mkdirSync(outputDir, { recursive: true });
+
+  // Resolve SDK directory for the owletto alias.
+  const sdkEntry = join(import.meta.dirname, 'index.ts');
+
+  const manifest: ManifestEntry[] = [];
+
+  for (const entry of entries) {
+    const outFile = join(outputDir, `${entry.name}.js`);
+
+    try {
+      await build({
+        entryPoints: [entry.entryPoint],
+        bundle: true,
+        format: 'esm',
+        platform: 'node',
+        target: 'node20',
+        outfile: outFile,
+        alias: {
+          owletto: sdkEntry,
+        },
+        banner: {
+          js: `import { createRequire as __createRequire } from 'module'; const require = __createRequire(import.meta.url);`,
+        },
+        external: ['playwright'],
+        plugins: [npmSpecifierPlugin()],
+        minify: false,
+        sourcemap: false,
+      });
+      console.log(`Built: ${entry.name} -> ${outFile}`);
+    } catch (err) {
+      console.error(`Failed to build ${entry.name}:`, err);
+      process.exit(1);
+    }
+
+    // Extract metadata: try the real bundle first, fall back to a stub build
+    // that replaces all runtime deps with empty modules (we only need to
+    // instantiate the class and read .definition, not run sync/execute).
+    let def: Omit<ManifestEntry, 'file'> | null = null;
+    try {
+      def = await extractDefinitionFromBundle(outFile);
+    } catch {
+      // Real bundle failed (missing npm deps) — try stub build
+      try {
+        def = await extractDefinitionViaStubBuild(
+          entry.entryPoint,
+          outputDir,
+          sdkEntry,
+          entry.name
+        );
+      } catch (stubErr) {
+        console.error(`  Failed to extract metadata from ${entry.name}:`, stubErr);
+        process.exit(1);
+      }
+    }
+
+    if (def) {
+      manifest.push({ file: `${entry.name}.js`, ...def });
+      console.log(`  Extracted metadata: ${def.key}@${def.version}`);
+    } else {
+      console.warn(
+        `  Warning: no ConnectorRuntime class found in ${entry.name}, skipping metadata`
+      );
+    }
+  }
+
+  // Discover insight template JSON files from insights/ directories
+  // (sibling to each connector input path)
+  const insightEntries: ManifestInsightEntry[] = [];
+  const seenInsightsDir = new Set<string>();
+
+  for (const inputPath of inputPaths) {
+    const abs = resolve(inputPath);
+    const parentDir = statSync(abs).isDirectory() ? resolve(abs, '..') : dirname(abs);
+    const insightsDir = join(parentDir, 'insights');
+
+    if (seenInsightsDir.has(insightsDir) || !existsSync(insightsDir)) continue;
+    seenInsightsDir.add(insightsDir);
+
+    for (const child of readdirSync(insightsDir, { withFileTypes: true })) {
+      if (!child.isFile() || !child.name.endsWith('.json')) continue;
+
+      const srcFile = join(insightsDir, child.name);
+      const destFile = join(outputDir, child.name);
+
+      try {
+        const content = JSON.parse(readFileSync(srcFile, 'utf-8')) as Record<string, unknown>;
+
+        if (!content.slug || !content.name || !content.version || !content.prompt) {
+          console.warn(
+            `  Skipping ${child.name}: missing required fields (slug, name, version, prompt)`
+          );
+          continue;
+        }
+
+        const sourceTypes = (content.source_types ?? {}) as {
+          required?: string[];
+          recommended?: string[];
+        };
+
+        insightEntries.push({
+          slug: content.slug as string,
+          file: child.name,
+          name: content.name as string,
+          description: typeof content.description === 'string' ? content.description : undefined,
+          version: content.version as number,
+          source_types: {
+            required: Array.isArray(sourceTypes.required) ? sourceTypes.required : [],
+            recommended: Array.isArray(sourceTypes.recommended) ? sourceTypes.recommended : [],
+          },
+        });
+
+        copyFileSync(srcFile, destFile);
+        console.log(`  Insight template: ${content.slug} v${content.version}`);
+      } catch (err) {
+        console.warn(
+          `  Skipping ${child.name}: ${err instanceof Error ? err.message : String(err)}`
+        );
+      }
+    }
+  }
+
+  const releaseManifest: ReleaseManifest = {
+    connectors: manifest,
+    insights: insightEntries,
+  };
+
+  const manifestPath = join(outputDir, 'manifest.json');
+  writeFileSync(manifestPath, JSON.stringify(releaseManifest, null, 2));
+
+  console.log(
+    `\nCompiled ${manifest.length} connector(s), ${insightEntries.length} insight template(s). Manifest: ${manifestPath}`
+  );
+}
+
+// --- CLI entry ---
+
+function parseArgs(argv: string[]) {
+  let outdir = 'dist';
+  const paths: string[] = [];
+
+  let i = 0;
+  while (i < argv.length) {
+    if (argv[i] === '--outdir' || argv[i] === '-o') {
+      outdir = argv[++i] || 'dist';
+    } else {
+      paths.push(argv[i]);
+    }
+    i++;
+  }
+
+  return { outdir, paths };
+}
+
+const allArgs = process.argv.slice(2);
+const command = allArgs[0];
+
+if (command === 'compile') {
+  const { outdir, paths } = parseArgs(allArgs.slice(1));
+
+  if (paths.length === 0) {
+    paths.push('.');
+  }
+
+  compile(paths, outdir).catch((err) => {
+    console.error('Compile failed:', err);
+    process.exit(1);
+  });
+} else {
+  console.log('Usage: owletto-sdk compile [options] <files or dirs...>');
+  console.log('');
+  console.log('Commands:');
+  console.log('  compile     Compile connector(s) and generate manifest.json');
+  console.log('');
+  console.log('Options:');
+  console.log('  --outdir, -o <dir>  Output directory (default: "dist")');
+  console.log('');
+  console.log('Examples:');
+  console.log('  owletto-sdk compile connectors/reddit.ts');
+  console.log('  owletto-sdk compile connectors/*.ts');
+  console.log('  owletto-sdk compile connectors/');
+  console.log('  owletto-sdk compile -o build connectors/*.ts');
+  if (command && command !== '--help' && command !== '-h') {
+    process.exit(1);
+  }
+}

package/src/connector-runtime.ts

@@ -0,0 +1,63 @@
+/**
+ * Connector Runtime
+ *
+ * Abstract base class that all connectors must implement.
+ * Provides the contract for sync (read) and execute (write) operations.
+ */
+
+import type {
+  ActionContext,
+  ActionResult,
+  ConnectorDefinition,
+  SyncContext,
+  SyncResult,
+} from './connector-types.js';
+
+/**
+ * ConnectorRuntime is the base class for all connectors.
+ *
+ * Subclasses must:
+ * - Set `definition` with connector metadata
+ * - Implement `sync()` for feed data ingestion
+ * - Implement `execute()` for action execution
+ *
+ * @example
+ * ```ts
+ * class GmailConnector extends ConnectorRuntime {
+ *   definition = { key: 'google.gmail', name: 'Gmail', version: '1.0.0', ... };
+ *
+ *   async sync(ctx: SyncContext): Promise<SyncResult> {
+ *     // Fetch threads from Gmail API
+ *   }
+ *
+ *   async execute(ctx: ActionContext): Promise<ActionResult> {
+ *     // Create draft, send email, etc.
+ *   }
+ * }
+ * ```
+ */
+export abstract class ConnectorRuntime {
+  /** Connector definition with metadata, feed schemas, and action schemas */
+  abstract readonly definition: ConnectorDefinition;
+
+  /**
+   * Sync data from the connected service.
+   *
+   * Called by the worker when a sync run is executed.
+   * Should return events to ingest and an updated checkpoint.
+   *
+   * @param ctx - Sync context with feed config, checkpoint, and credentials
+   * @returns Events and updated checkpoint
+   */
+  abstract sync(ctx: SyncContext): Promise<SyncResult>;
+
+  /**
+   * Execute an action on the connected service.
+   *
+   * Called either inline (low-risk) or by the worker (high-risk with approval).
+   *
+   * @param ctx - Action context with action key, input, and credentials
+   * @returns Action result with output data
+   */
+  abstract execute(ctx: ActionContext): Promise<ActionResult>;
+}