@dot-ai/core 0.10.0 → 0.11.0

package/src/extension-loader.ts

@@ -5,77 +5,198 @@ import type { ExtensionAPI } from './extension-api.js';
 import type { LoadedExtension, ToolDefinition, CommandDefinition } from './extension-types.js';
 import type { ExtensionsConfig, Skill, Identity } from './types.js';
 
+function isExtensionFile(name: string): boolean {
+  return name.endsWith('.ts') || name.endsWith('.js');
+}
+
+/**
+ * Resolve extension entry points from a directory.
+ *
+ * Checks for:
+ * 1. package.json with "dot-ai.extensions" field -> returns declared paths
+ * 2. index.ts or index.js -> returns the index file
+ *
+ * Returns resolved paths or null if no entry points found.
+ */
+async function resolveExtensionEntries(dir: string): Promise<string[] | null> {
+  // Check for package.json with "dot-ai" field first
+  try {
+    const pkgPath = join(dir, 'package.json');
+    const pkgRaw = await readFile(pkgPath, 'utf-8');
+    const pkg = JSON.parse(pkgRaw) as Record<string, unknown>;
+    const dotAi = pkg['dot-ai'] as { extensions?: string[] } | undefined;
+    if (dotAi?.extensions && Array.isArray(dotAi.extensions)) {
+      const entries: string[] = [];
+      for (const ext of dotAi.extensions) {
+        const resolvedPath = resolve(dir, ext);
+        try {
+          await stat(resolvedPath);
+          entries.push(resolvedPath);
+        } catch { /* entry doesn't exist */ }
+      }
+      if (entries.length > 0) return entries;
+    }
+  } catch { /* no package.json or invalid */ }
+
+  // Check for index.ts or index.js
+  for (const indexName of ['index.ts', 'index.js']) {
+    const indexPath = join(dir, indexName);
+    try {
+      await stat(indexPath);
+      return [indexPath];
+    } catch { /* not found */ }
+  }
+
+  return null;
+}
+
+/**
+ * Scan a directory for extensions.
+ *
+ * Discovery rules:
+ * 1. Direct files: `dir/*.ts` or `*.js` -> load
+ * 2. Subdirectory with package.json: `dir/sub/package.json` with "dot-ai" field -> load what it declares
+ * 3. Subdirectory with index: `dir/sub/index.ts` or `index.js` -> load
+ *
+ * No recursion beyond one level. Complex packages must use package.json manifest.
+ */
+async function discoverExtensionsInDir(dir: string): Promise<string[]> {
+  const discovered: string[] = [];
+
+  try {
+    const entries = await readdir(dir, { withFileTypes: true });
+
+    for (const entry of entries) {
+      const entryPath = join(dir, entry.name);
+
+      // 1. Direct files: *.ts or *.js
+      if (entry.isFile() && isExtensionFile(entry.name)) {
+        discovered.push(entryPath);
+        continue;
+      }
+
+      // 2 & 3. Subdirectories: package.json or index file
+      if (entry.isDirectory()) {
+        const resolved = await resolveExtensionEntries(entryPath);
+        if (resolved) {
+          discovered.push(...resolved);
+        }
+      }
+    }
+  } catch { /* directory doesn't exist — skip */ }
+
+  return discovered;
+}
+
+/**
+ * Resolve extensions from installed packages in .ai/packages/.
+ *
+ * Reads the package.json created by `npm --prefix .ai/packages/` and resolves
+ * extension entry points from each installed package's "dot-ai.extensions" field.
+ */
+async function resolveInstalledPackages(workspaceRoot: string): Promise<string[]> {
+  const packagesDir = join(workspaceRoot, '.ai', 'packages');
+  const pkgJsonPath = join(packagesDir, 'package.json');
+
+  try {
+    const raw = await readFile(pkgJsonPath, 'utf-8');
+    const pkg = JSON.parse(raw) as Record<string, unknown>;
+    const deps = pkg.dependencies as Record<string, string> | undefined;
+    if (!deps) return [];
+
+    const paths: string[] = [];
+    for (const name of Object.keys(deps)) {
+      const pkgDir = join(packagesDir, 'node_modules', name);
+      const entries = await resolveExtensionEntries(pkgDir);
+      if (entries) {
+        paths.push(...entries);
+      }
+    }
+    return paths;
+  } catch {
+    return [];
+  }
+}
+
 /**
- * Discover extension file paths from configured locations.
+ * Discover extension file paths from all sources.
+ *
+ * Sources (in order):
+ * 1. Auto-discovery: .ai/extensions/ (project-local)
+ * 2. Auto-discovery: ~/.ai/extensions/ (global)
+ * 3. Installed packages: .ai/packages/node_modules/ (installed via dot-ai install)
+ * 4. Configured paths: settings.json "extensions" array (explicit paths/dirs)
+ * 5. Configured packages: settings.json "packages" array (npm package names resolved from workspace node_modules)
  */
 export async function discoverExtensions(
   workspaceRoot: string,
   config?: ExtensionsConfig,
 ): Promise<string[]> {
-  const paths = new Set<string>();
+  const seen = new Set<string>();
+  const allPaths: string[] = [];
+
+  const addPaths = (paths: string[]) => {
+    for (const p of paths) {
+      const resolved = resolve(p);
+      if (!seen.has(resolved)) {
+        seen.add(resolved);
+        allPaths.push(resolved);
+      }
+    }
+  };
 
-  // Default discovery paths
-  const searchDirs = [
-    join(workspaceRoot, '.ai', 'extensions'),
-    join(homedir(), '.ai', 'extensions'),
-    ...(config?.paths ?? []).map(p => resolve(workspaceRoot, p)),
-  ];
+  // 1. Project-local extensions: .ai/extensions/
+  addPaths(await discoverExtensionsInDir(join(workspaceRoot, '.ai', 'extensions')));
 
-  for (const dir of searchDirs) {
-    try {
-      const entries = await readdir(dir, { withFileTypes: true });
-      for (const entry of entries) {
-        const fullPath = join(dir, entry.name);
-        if (entry.isFile() && (entry.name.endsWith('.ts') || entry.name.endsWith('.js'))) {
-          paths.add(fullPath);
-        } else if (entry.isDirectory()) {
-          // Check for index.ts or index.js
-          for (const indexName of ['index.ts', 'index.js']) {
-            const indexPath = join(fullPath, indexName);
-            try {
-              await stat(indexPath);
-              paths.add(indexPath);
-              break;
-            } catch { /* not found */ }
+  // 2. Global extensions: ~/.ai/extensions/
+  addPaths(await discoverExtensionsInDir(join(homedir(), '.ai', 'extensions')));
+
+  // 3. Installed packages: .ai/packages/node_modules/
+  addPaths(await resolveInstalledPackages(workspaceRoot));
+
+  // 4. Explicitly configured paths from settings.json "extensions" array
+  if (config?.paths) {
+    for (const p of config.paths) {
+      const resolved = resolve(workspaceRoot, p);
+      try {
+        const s = await stat(resolved);
+        if (s.isDirectory()) {
+          // Check for package.json or index file first
+          const entries = await resolveExtensionEntries(resolved);
+          if (entries) {
+            addPaths(entries);
+          } else {
+            // Discover individual files in directory
+            addPaths(await discoverExtensionsInDir(resolved));
           }
-          // Check for package.json with dot-ai field
-          try {
-            const pkgPath = join(fullPath, 'package.json');
-            const pkgRaw = await readFile(pkgPath, 'utf-8');
-            const pkg = JSON.parse(pkgRaw) as Record<string, unknown>;
-            const dotAi = pkg['dot-ai'] as { extensions?: string[] } | undefined;
-            if (dotAi?.extensions && Array.isArray(dotAi.extensions)) {
-              for (const ext of dotAi.extensions) {
-                paths.add(resolve(fullPath, ext));
-              }
-            }
-          } catch { /* no package.json or no dot-ai field */ }
+        } else if (s.isFile()) {
+          addPaths([resolved]);
         }
+      } catch {
+        // Path doesn't exist — add anyway (will fail at load time with clear error)
+        addPaths([resolved]);
       }
-    } catch { /* directory doesn't exist — skip */ }
+    }
   }
 
-  // Also resolve npm packages from config
+  // 5. Configured npm packages from settings.json "packages" array
+  // These are resolved from the workspace's own node_modules
   if (config?.packages) {
     for (const pkg of config.packages) {
       try {
         const { createRequire } = await import('node:module');
         const require = createRequire(join(workspaceRoot, 'package.json'));
         const pkgJsonPath = require.resolve(`${pkg}/package.json`);
-        const pkgRaw = await readFile(pkgJsonPath, 'utf-8');
-        const pkgJson = JSON.parse(pkgRaw) as Record<string, unknown>;
-        const dotAi = pkgJson['dot-ai'] as { extensions?: string[] } | undefined;
-        if (dotAi?.extensions && Array.isArray(dotAi.extensions)) {
-          const pkgDir = join(pkgJsonPath, '..');
-          for (const ext of dotAi.extensions) {
-            paths.add(resolve(pkgDir, ext));
-          }
+        const pkgDir = join(pkgJsonPath, '..');
+        const entries = await resolveExtensionEntries(pkgDir);
+        if (entries) {
+          addPaths(entries);
         }
       } catch { /* package not found */ }
     }
   }
 
-  return Array.from(paths);
+  return allPaths;
 }
 
 /**

package/src/index.ts

@@ -85,8 +85,8 @@ export { NoopLogger, JsonFileLogger, StderrLogger } from './logger.js';
 export { loadConfig, migrateConfig } from './config.js';
 
 // ── Package Manager ──
-export { install, remove, listPackages, resolvePackages } from './package-manager.js';
-export type { PackageInfo } from './package-manager.js';
+export { install, remove, listPackages, resolvePackages, ensurePackagesInstalled } from './package-manager.js';
+export type { PackageInfo, MissingPackageAction } from './package-manager.js';
 
 // ── Boot Cache ──
 export { computeChecksum, loadBootCache, writeBootCache, clearBootCache } from './boot-cache.js';

package/src/package-manager.ts

@@ -1,6 +1,7 @@
 import { execSync } from 'node:child_process';
-import { readFile, mkdir } from 'node:fs/promises';
-import { join } from 'node:path';
+import { readFile, mkdir, writeFile } from 'node:fs/promises';
+import { existsSync } from 'node:fs';
+import { join, resolve } from 'node:path';
 
 export interface PackageInfo {
   name: string;
@@ -12,25 +13,83 @@ export interface PackageInfo {
   };
 }
 
+export type MissingPackageAction = 'install' | 'skip' | 'error';
+
 /**
- * Install a dot-ai package from npm or git.
+ * Parse a package source string into name and spec.
+ *
+ * Supported formats:
+ * - "npm:@dot-ai/ext-memory@1.0.0" → { name: "@dot-ai/ext-memory", spec: "@dot-ai/ext-memory@1.0.0" }
+ * - "npm:@dot-ai/ext-memory" → { name: "@dot-ai/ext-memory", spec: "@dot-ai/ext-memory" }
+ * - "@dot-ai/ext-memory@1.0.0" → { name: "@dot-ai/ext-memory", spec: "@dot-ai/ext-memory@1.0.0" }
+ * - "@dot-ai/ext-memory" → { name: "@dot-ai/ext-memory", spec: "@dot-ai/ext-memory" }
  */
-export async function install(
-  source: string,
-  targetDir: string,
-): Promise<PackageInfo> {
-  const installDir = join(targetDir, '.ai', 'packages');
+function parsePackageSource(source: string): { name: string; spec: string } {
+  // Strip npm: prefix
+  const raw = source.startsWith('npm:') ? source.slice(4) : source;
+
+  // Extract name (without version) for scoped packages
+  if (raw.startsWith('@')) {
+    // @scope/name@version → name = @scope/name
+    const slashIdx = raw.indexOf('/');
+    if (slashIdx === -1) return { name: raw, spec: raw };
+    const afterSlash = raw.slice(slashIdx + 1);
+    const atIdx = afterSlash.indexOf('@');
+    if (atIdx === -1) return { name: raw, spec: raw };
+    return { name: raw.slice(0, slashIdx + 1 + atIdx), spec: raw };
+  }
+
+  // name@version → name = name
+  const atIdx = raw.indexOf('@');
+  if (atIdx === -1) return { name: raw, spec: raw };
+  return { name: raw.slice(0, atIdx), spec: raw };
+}
+
+/**
+ * Get the install directory for packages.
+ */
+function getInstallDir(workspaceRoot: string): string {
+  return join(workspaceRoot, '.ai', 'packages');
+}
+
+/**
+ * Check if a package is already installed in .ai/packages/.
+ */
+function isPackageInstalled(installDir: string, name: string): boolean {
+  return existsSync(join(installDir, 'node_modules', name, 'package.json'));
+}
+
+/**
+ * Install a single npm package into .ai/packages/.
+ */
+async function installNpm(spec: string, installDir: string): Promise<void> {
   await mkdir(installDir, { recursive: true });
 
-  execSync(`npm install --prefix "${installDir}" "${source}"`, {
+  // Ensure package.json exists (npm --prefix needs it)
+  const pkgJsonPath = join(installDir, 'package.json');
+  if (!existsSync(pkgJsonPath)) {
+    await writeFile(pkgJsonPath, JSON.stringify({ private: true, dependencies: {} }, null, 2) + '\n');
+  }
+
+  execSync(`npm install --prefix "${installDir}" "${spec}"`, {
     stdio: 'pipe',
     timeout: 60000,
   });
+}
+
+/**
+ * Install a dot-ai package from npm.
+ *
+ * Installs into `.ai/packages/node_modules/`.
+ */
+export async function install(
+  source: string,
+  workspaceRoot: string,
+): Promise<PackageInfo> {
+  const installDir = getInstallDir(workspaceRoot);
+  const { name, spec } = parsePackageSource(source);
 
-  // Read installed package info
-  const name = source.startsWith('@') || !source.includes('/')
-    ? source.replace(/@[^/]*$/, '')
-    : source;
+  await installNpm(spec, installDir);
   return readPackageInfo(installDir, name);
 }
 
@@ -39,9 +98,9 @@ export async function install(
  */
 export async function remove(
   name: string,
-  targetDir: string,
+  workspaceRoot: string,
 ): Promise<void> {
-  const installDir = join(targetDir, '.ai', 'packages');
+  const installDir = getInstallDir(workspaceRoot);
   execSync(`npm uninstall --prefix "${installDir}" "${name}"`, {
     stdio: 'pipe',
     timeout: 60000,
@@ -52,9 +111,9 @@ export async function remove(
  * List installed dot-ai packages.
  */
 export async function listPackages(
-  targetDir: string,
+  workspaceRoot: string,
 ): Promise<PackageInfo[]> {
-  const installDir = join(targetDir, '.ai', 'packages');
+  const installDir = getInstallDir(workspaceRoot);
   const pkgJsonPath = join(installDir, 'package.json');
 
   try {
@@ -82,24 +141,88 @@ export async function listPackages(
  * Resolve dot-ai manifest from installed packages.
  */
 export async function resolvePackages(
-  targetDir: string,
+  workspaceRoot: string,
 ): Promise<{ extensions: string[]; skills: string[]; providers: string[] }> {
-  const packages = await listPackages(targetDir);
+  const packages = await listPackages(workspaceRoot);
   const result = { extensions: [] as string[], skills: [] as string[], providers: [] as string[] };
 
-  const installDir = join(targetDir, '.ai', 'packages');
+  const installDir = getInstallDir(workspaceRoot);
   for (const pkg of packages) {
     if (!pkg.dotAi) continue;
     const pkgDir = join(installDir, 'node_modules', pkg.name);
 
     if (pkg.dotAi.extensions) {
-      result.extensions.push(...pkg.dotAi.extensions.map(e => join(pkgDir, e)));
+      result.extensions.push(...pkg.dotAi.extensions.map(e => resolve(pkgDir, e)));
     }
     if (pkg.dotAi.skills) {
-      result.skills.push(...pkg.dotAi.skills.map(s => join(pkgDir, s)));
+      result.skills.push(...pkg.dotAi.skills.map(s => resolve(pkgDir, s)));
     }
     if (pkg.dotAi.providers) {
-      result.providers.push(...pkg.dotAi.providers.map(p => join(pkgDir, p)));
+      result.providers.push(...pkg.dotAi.providers.map(p => resolve(pkgDir, p)));
+    }
+  }
+
+  return result;
+}
+
+/**
+ * Ensure all packages from settings.json are installed.
+ *
+ * Like Pi's `resolvePackageSources()` with auto-install:
+ * - Reads the `packages` array from settings.json config
+ * - For each package, checks if it's already installed in `.ai/packages/`
+ * - If not installed, auto-installs it via npm
+ * - The `onMissing` callback allows callers to control behavior (install/skip/error)
+ *
+ * This should be called BEFORE `discoverExtensions()` in the boot flow,
+ * so that installed packages are available for extension discovery.
+ */
+export async function ensurePackagesInstalled(
+  workspaceRoot: string,
+  packages: string[],
+  onMissing?: (source: string) => Promise<MissingPackageAction>,
+): Promise<{ installed: string[]; skipped: string[]; errors: Array<{ source: string; error: string }> }> {
+  const installDir = getInstallDir(workspaceRoot);
+  const result = {
+    installed: [] as string[],
+    skipped: [] as string[],
+    errors: [] as Array<{ source: string; error: string }>,
+  };
+
+  for (const source of packages) {
+    const { name, spec } = parsePackageSource(source);
+
+    // Already installed? Skip.
+    if (isPackageInstalled(installDir, name)) {
+      result.skipped.push(source);
+      continue;
+    }
+
+    // Determine action for missing package
+    let action: MissingPackageAction = 'install';
+    if (onMissing) {
+      action = await onMissing(source);
+    }
+
+    if (action === 'skip') {
+      result.skipped.push(source);
+      continue;
+    }
+
+    if (action === 'error') {
+      result.errors.push({ source, error: `Missing package: ${source}` });
+      continue;
+    }
+
+    // Auto-install
+    try {
+      await installNpm(spec, installDir);
+      result.installed.push(source);
+    } catch (err) {
+      result.errors.push({
+        source,
+        error: err instanceof Error ? err.message : String(err),
+      });
     }
   }
 

package/src/runtime.ts

@@ -2,6 +2,7 @@ import { loadConfig } from './config.js';
 import { toolDefinitionToCapability } from './capabilities.js';
 import type { Capability } from './capabilities.js';
 import { discoverExtensions, createV6CollectorAPI } from './extension-loader.js';
+import { ensurePackagesInstalled } from './package-manager.js';
 import { ExtensionRunner, EventBus } from './extension-runner.js';
 import type {
   ToolCallEvent, ToolCallResult, ExtensionDiagnostic,
@@ -92,8 +93,29 @@ export class DotAiRuntime {
     // Create event bus
     this._eventBus = new EventBus();
 
-    // Discover extensions
+    // Auto-install packages from settings.json before discovery
     const extConfig = this.options.extensions ?? rawConfig.extensions;
+    if (extConfig?.packages?.length) {
+      const installResult = await ensurePackagesInstalled(
+        this.options.workspaceRoot,
+        extConfig.packages,
+      );
+      if (installResult.installed.length > 0 || installResult.errors.length > 0) {
+        this.logger.log({
+          timestamp: new Date().toISOString(),
+          level: installResult.errors.length > 0 ? 'warn' : 'info',
+          phase: 'boot',
+          event: 'packages_installed',
+          data: {
+            installed: installResult.installed,
+            skipped: installResult.skipped,
+            errors: installResult.errors,
+          },
+        });
+      }
+    }
+
+    // Discover extensions
     const extPaths = await discoverExtensions(this.options.workspaceRoot, extConfig);
 
     // Load extensions via collector API