macro-agent 0.1.0 → 0.1.2

package/references/skill-tree/scraper/docs/DESIGN.md

@@ -0,0 +1,683 @@
+# Skills Indexer Architecture Design
+
+## System Overview
+
+```
+┌─────────────────────────────────────────────────────────────────────────┐
+│                           Skills Indexer                                 │
+├─────────────────────────────────────────────────────────────────────────┤
+│                                                                          │
+│  ┌──────────────┐    ┌──────────────┐    ┌──────────────┐               │
+│  │   CLI        │───▶│   Scraper    │───▶│   Indexer    │               │
+│  │   Interface  │    │   Agent      │    │   Agent      │               │
+│  └──────────────┘    └──────────────┘    └──────────────┘               │
+│         │                   │                   │                        │
+│         │                   ▼                   ▼                        │
+│         │            ┌──────────────┐    ┌──────────────┐               │
+│         │            │   GitHub     │    │   Claude     │               │
+│         │            │   API        │    │   API        │               │
+│         │            └──────────────┘    └──────────────┘               │
+│         │                   │                   │                        │
+│         ▼                   ▼                   ▼                        │
+│  ┌─────────────────────────────────────────────────────────────────┐    │
+│  │                     Database Adapter                             │    │
+│  │  ┌─────────────────┐              ┌─────────────────┐           │    │
+│  │  │  SQLite         │              │  PostgreSQL     │           │    │
+│  │  │  (default)      │              │  (future)       │           │    │
+│  │  └─────────────────┘              └─────────────────┘           │    │
+│  └─────────────────────────────────────────────────────────────────┘    │
+│                                                                          │
+└─────────────────────────────────────────────────────────────────────────┘
+```
+
+## Core Design Principles
+
+### 1. Async-First Architecture
+
+All I/O operations are async to enable:
+- Parallel scraping of multiple repositories
+- Concurrent AI classification requests
+- Non-blocking database operations
+- Responsive CLI experience
+
+```typescript
+// All service methods return Promises
+interface ScraperService {
+  scrapeRepository(url: string): Promise<ScrapedSkill[]>;
+  discoverFromAwesomeList(url: string): Promise<string[]>;
+}
+
+interface IndexerService {
+  classifySkill(skill: RawSkill): Promise<Classification>;
+  batchClassify(skills: RawSkill[]): Promise<Classification[]>;
+}
+```
+
+### 2. Adapter Pattern for Database
+
+Abstract database operations behind an interface to enable:
+- SQLite for local development and small deployments
+- PostgreSQL for production scale and vector search
+- Easy testing with in-memory implementations
+
+```typescript
+interface DatabaseAdapter {
+  // Connection lifecycle
+  connect(): Promise<void>;
+  disconnect(): Promise<void>;
+
+  // Skills
+  insertSkill(skill: Skill): Promise<string>;
+  getSkill(id: string): Promise<Skill | null>;
+  listSkills(filter?: SkillFilter): Promise<Skill[]>;
+  updateSkill(id: string, updates: Partial<Skill>): Promise<void>;
+  deleteSkill(id: string): Promise<void>;
+
+  // Taxonomy
+  insertTaxonomyNode(node: TaxonomyNode): Promise<string>;
+  getTaxonomyTree(rootPath?: string): Promise<TaxonomyTree>;
+  updateTaxonomyNode(id: string, updates: Partial<TaxonomyNode>): Promise<void>;
+
+  // Placements
+  placeSkill(skillId: string, path: string[], isPrimary: boolean): Promise<void>;
+  getSkillPlacements(skillId: string): Promise<SkillPlacement[]>;
+  getSkillsAtPath(path: string[]): Promise<Skill[]>;
+
+  // Search
+  searchSkills(query: string): Promise<Skill[]>;
+
+  // Migrations
+  migrate(): Promise<void>;
+}
+```
+
+### 3. Agent-Based Processing
+
+Specialized agents handle discrete tasks:
+
+```typescript
+// Scraper Agent - discovers and fetches skills
+interface ScraperAgent {
+  discover(sources: SkillSource[]): AsyncGenerator<DiscoveredSkill>;
+  fetch(skill: DiscoveredSkill): Promise<RawSkill>;
+}
+
+// Indexer Agent - classifies and organizes skills
+interface IndexerAgent {
+  classify(skill: RawSkill): Promise<Classification>;
+  suggestTaxonomy(skills: RawSkill[]): Promise<TaxonomySuggestion>;
+  detectRelationships(skills: RawSkill[]): Promise<Relationship[]>;
+}
+```
+
+## Data Models
+
+### Skill (Parsed from SKILL.md)
+
+```typescript
+interface Skill {
+  // Identity
+  id: string;                    // Generated UUID
+  slug: string;                  // From _meta.json or derived
+  name: string;                  // From frontmatter
+  displayName?: string;          // From _meta.json
+
+  // Source
+  sourceRepo: string;            // e.g., "openclaw/skills"
+  sourcePath: string;            // e.g., "skills/username/skill-name"
+  sourceUrl: string;             // Full GitHub URL
+  author: string;                // Directory name or frontmatter
+
+  // Content
+  description: string;           // From frontmatter
+  content: string;               // Full markdown content
+
+  // Metadata
+  version: string;               // From _meta.json or "1.0.0"
+  homepage?: string;             // From frontmatter
+  metadata?: Record<string, unknown>; // Platform-specific metadata
+
+  // Indexing
+  status: 'raw' | 'indexed' | 'failed';
+  indexedAt?: Date;
+  indexingError?: string;
+
+  // Timestamps
+  scrapedAt: Date;
+  updatedAt: Date;
+  publishedAt?: Date;            // From _meta.json
+}
+```
+
+### Taxonomy Node
+
+```typescript
+interface TaxonomyNode {
+  id: string;
+  name: string;                  // e.g., "Python"
+  path: string[];                // e.g., ["Development", "Languages", "Python"]
+  fullPath: string;              // e.g., "Development/Languages/Python"
+
+  // Hierarchy
+  parentId: string | null;
+  depth: number;
+  childCount: number;
+  skillCount: number;            // Skills directly in this node
+
+  // AI reasoning
+  createdBy: 'ai' | 'manual';
+  creationReasoning?: string;    // Why this category exists
+
+  // Timestamps
+  createdAt: Date;
+  updatedAt: Date;
+}
+```
+
+### Skill Placement
+
+```typescript
+interface SkillPlacement {
+  id: string;
+  skillId: string;
+  taxonomyNodeId: string;
+
+  // Classification
+  isPrimary: boolean;            // Primary path vs secondary
+  confidence: number;            // 0-1 classification confidence
+  reasoning: string;             // AI's reasoning for placement
+
+  // Metadata
+  placedAt: Date;
+  placedBy: 'ai' | 'manual';
+}
+```
+
+### Relationship
+
+```typescript
+interface SkillRelationship {
+  id: string;
+  sourceSkillId: string;
+  targetSkillId: string;
+
+  type: 'depends_on' | 'extends' | 'alternative' | 'related';
+  strength: number;              // 0-1 relationship strength
+  reasoning: string;
+
+  detectedAt: Date;
+  detectedBy: 'ai' | 'manual';
+}
+```
+
+## Database Schema (Drizzle ORM)
+
+```typescript
+// schema.ts
+import { sqliteTable, text, integer, real } from 'drizzle-orm/sqlite-core';
+
+export const skills = sqliteTable('skills', {
+  id: text('id').primaryKey(),
+  slug: text('slug').notNull().unique(),
+  name: text('name').notNull(),
+  displayName: text('display_name'),
+
+  sourceRepo: text('source_repo').notNull(),
+  sourcePath: text('source_path').notNull(),
+  sourceUrl: text('source_url').notNull(),
+  author: text('author').notNull(),
+
+  description: text('description').notNull(),
+  content: text('content').notNull(),
+
+  version: text('version').notNull().default('1.0.0'),
+  homepage: text('homepage'),
+  metadata: text('metadata', { mode: 'json' }),
+
+  status: text('status', { enum: ['raw', 'indexed', 'failed'] }).notNull().default('raw'),
+  indexedAt: integer('indexed_at', { mode: 'timestamp' }),
+  indexingError: text('indexing_error'),
+
+  scrapedAt: integer('scraped_at', { mode: 'timestamp' }).notNull(),
+  updatedAt: integer('updated_at', { mode: 'timestamp' }).notNull(),
+  publishedAt: integer('published_at', { mode: 'timestamp' }),
+});
+
+export const taxonomyNodes = sqliteTable('taxonomy_nodes', {
+  id: text('id').primaryKey(),
+  name: text('name').notNull(),
+  path: text('path', { mode: 'json' }).$type<string[]>().notNull(),
+  fullPath: text('full_path').notNull().unique(),
+
+  parentId: text('parent_id').references(() => taxonomyNodes.id),
+  depth: integer('depth').notNull(),
+  childCount: integer('child_count').notNull().default(0),
+  skillCount: integer('skill_count').notNull().default(0),
+
+  createdBy: text('created_by', { enum: ['ai', 'manual'] }).notNull(),
+  creationReasoning: text('creation_reasoning'),
+
+  createdAt: integer('created_at', { mode: 'timestamp' }).notNull(),
+  updatedAt: integer('updated_at', { mode: 'timestamp' }).notNull(),
+});
+
+export const skillPlacements = sqliteTable('skill_placements', {
+  id: text('id').primaryKey(),
+  skillId: text('skill_id').notNull().references(() => skills.id),
+  taxonomyNodeId: text('taxonomy_node_id').notNull().references(() => taxonomyNodes.id),
+
+  isPrimary: integer('is_primary', { mode: 'boolean' }).notNull(),
+  confidence: real('confidence').notNull(),
+  reasoning: text('reasoning').notNull(),
+
+  placedAt: integer('placed_at', { mode: 'timestamp' }).notNull(),
+  placedBy: text('placed_by', { enum: ['ai', 'manual'] }).notNull(),
+});
+
+export const skillRelationships = sqliteTable('skill_relationships', {
+  id: text('id').primaryKey(),
+  sourceSkillId: text('source_skill_id').notNull().references(() => skills.id),
+  targetSkillId: text('target_skill_id').notNull().references(() => skills.id),
+
+  type: text('type', {
+    enum: ['depends_on', 'extends', 'alternative', 'related']
+  }).notNull(),
+  strength: real('strength').notNull(),
+  reasoning: text('reasoning').notNull(),
+
+  detectedAt: integer('detected_at', { mode: 'timestamp' }).notNull(),
+  detectedBy: text('detected_by', { enum: ['ai', 'manual'] }).notNull(),
+});
+
+export const scrapeState = sqliteTable('scrape_state', {
+  id: text('id').primaryKey(),
+  sourceUrl: text('source_url').notNull().unique(),
+  lastScrapedAt: integer('last_scraped_at', { mode: 'timestamp' }),
+  lastCommitSha: text('last_commit_sha'),
+  skillCount: integer('skill_count').notNull().default(0),
+  status: text('status', { enum: ['pending', 'in_progress', 'completed', 'failed'] }).notNull(),
+  error: text('error'),
+});
+```
+
+## AI Classification Design
+
+### Classification Prompt
+
+```typescript
+const CLASSIFICATION_PROMPT = `You are an expert at organizing software development skills into a hierarchical taxonomy.
+
+Given a skill definition, classify it into the most appropriate category path(s).
+
+## Skill to Classify
+
+Name: {name}
+Description: {description}
+
+Content:
+{content}
+
+## Current Taxonomy
+{taxonomyTree}
+
+## Instructions
+
+1. Determine the PRIMARY category path where this skill best fits
+2. Optionally suggest SECONDARY paths if the skill fits multiple categories
+3. If no existing category fits well, suggest a NEW category with reasoning
+4. Identify any relationships to other skills
+
+## Response Format (JSON)
+
+{
+  "primaryPath": ["Level1", "Level2", "Level3"],
+  "secondaryPaths": [["Alt1", "Alt2"]],
+  "confidence": 0.85,
+  "reasoning": "This skill is primarily about X because...",
+  "suggestedNewCategories": [
+    {
+      "path": ["New", "Category"],
+      "reasoning": "Why this category should exist"
+    }
+  ],
+  "relationships": [
+    {
+      "skillSlug": "other-skill",
+      "type": "extends",
+      "reasoning": "This skill extends other-skill by..."
+    }
+  ],
+  "suggestedTags": ["tag1", "tag2"]
+}`;
+```
+
+### Classification Response
+
+```typescript
+interface ClassificationResponse {
+  primaryPath: string[];
+  secondaryPaths?: string[][];
+  confidence: number;
+  reasoning: string;
+  suggestedNewCategories?: {
+    path: string[];
+    reasoning: string;
+  }[];
+  relationships?: {
+    skillSlug: string;
+    type: 'depends_on' | 'extends' | 'alternative' | 'related';
+    reasoning: string;
+  }[];
+  suggestedTags?: string[];
+}
+```
+
+## GitHub Scraping Design
+
+### Discovery Flow
+
+```
+1. Parse awesome-list README.md
+   ├── Extract repository links
+   ├── Extract direct skill links
+   └── Queue for scraping
+
+2. For each repository:
+   ├── Fetch repo metadata
+   ├── Get directory tree
+   ├── Find SKILL.md files
+   └── Queue skills for parsing
+
+3. For each skill:
+   ├── Fetch SKILL.md content
+   ├── Fetch _meta.json if exists
+   ├── Parse frontmatter + content
+   └── Store as RawSkill
+```
+
+### Rate Limit Handling
+
+```typescript
+interface RateLimitConfig {
+  requestsPerHour: number;       // GitHub: 5000 for authenticated
+  minDelayMs: number;            // Minimum delay between requests
+  maxRetries: number;            // Max retries on 429
+  backoffMultiplier: number;     // Exponential backoff multiplier
+}
+
+const DEFAULT_RATE_LIMIT: RateLimitConfig = {
+  requestsPerHour: 5000,
+  minDelayMs: 100,
+  maxRetries: 3,
+  backoffMultiplier: 2,
+};
+```
+
+### Caching Strategy
+
+```typescript
+interface CacheConfig {
+  enabled: boolean;
+  directory: string;             // Default: ./data/cache
+  ttlSeconds: number;            // Default: 86400 (24 hours)
+  maxSizeBytes: number;          // Default: 100MB
+}
+
+// Cache keys:
+// - repo:{owner}/{name}/tree - Repository file tree
+// - repo:{owner}/{name}/readme - README content
+// - skill:{owner}/{repo}/{path} - Individual skill content
+```
+
+## CLI Design
+
+### Command Structure
+
+```
+skillindexer <command> [options]
+
+Commands:
+  scrape <source>     Scrape skills from a source
+  index               Index unclassified skills
+  tree [path]         Browse taxonomy tree
+  list                List all skills
+  show <skill>        Show skill details
+  search <query>      Search skills
+  export              Export database
+  stats               Show statistics
+
+Global Options:
+  --config, -c        Path to config file
+  --verbose, -v       Verbose output
+  --quiet, -q         Minimal output
+  --help, -h          Show help
+  --version           Show version
+```
+
+### Output Formatting
+
+```typescript
+// Tree view
+Development/
+├── Languages/
+│   ├── Python/ (15 skills)
+│   │   ├── Testing/
+│   │   └── Web-Frameworks/
+│   └── JavaScript/ (23 skills)
+└── Infrastructure/
+    └── Docker/ (8 skills)
+
+// Skill detail view
+┌─────────────────────────────────────────────┐
+│ react-testing-helper                        │
+├─────────────────────────────────────────────┤
+│ Author: community                           │
+│ Version: 1.2.0                              │
+│ Source: openclaw/skills                     │
+├─────────────────────────────────────────────┤
+│ Description:                                │
+│ Helps write and debug React component tests │
+│ using React Testing Library.                │
+├─────────────────────────────────────────────┤
+│ Taxonomy:                                   │
+│ • Development/Frontend/React/Testing        │
+│ • Development/Testing/Unit-Testing          │
+├─────────────────────────────────────────────┤
+│ Tags: react, testing, frontend, rtl         │
+└─────────────────────────────────────────────┘
+```
+
+## Configuration
+
+### Environment Variables
+
+```bash
+# Required
+GITHUB_TOKEN=ghp_xxxxx              # GitHub personal access token
+ANTHROPIC_API_KEY=sk-ant-xxxxx      # Anthropic API key
+
+# Optional
+SKILLINDEXER_DB_PATH=./data/skills.db
+SKILLINDEXER_CACHE_DIR=./data/cache
+SKILLINDEXER_LOG_LEVEL=info         # debug, info, warn, error
+SKILLINDEXER_CLAUDE_MODEL=claude-sonnet-4-20250514
+```
+
+### Config File (skillindexer.config.ts)
+
+```typescript
+import { defineConfig } from './src/config';
+
+export default defineConfig({
+  database: {
+    type: 'sqlite',              // 'sqlite' | 'postgresql'
+    path: './data/skills.db',
+    // PostgreSQL options (future)
+    // host: 'localhost',
+    // port: 5432,
+    // database: 'skillindexer',
+  },
+
+  github: {
+    token: process.env.GITHUB_TOKEN,
+    cacheEnabled: true,
+    cacheTtlSeconds: 86400,
+  },
+
+  anthropic: {
+    apiKey: process.env.ANTHROPIC_API_KEY,
+    model: 'claude-sonnet-4-20250514',
+    maxTokens: 4096,
+  },
+
+  indexer: {
+    batchSize: 10,               // Skills per batch
+    concurrency: 3,              // Parallel classifications
+    minConfidence: 0.7,          // Minimum classification confidence
+  },
+
+  sources: [
+    'https://github.com/VoltAgent/awesome-agent-skills',
+    'https://github.com/VoltAgent/awesome-openclaw-skills',
+  ],
+});
+```
+
+## Error Handling
+
+### Error Types
+
+```typescript
+class SkillIndexerError extends Error {
+  constructor(
+    message: string,
+    public code: string,
+    public cause?: Error
+  ) {
+    super(message);
+    this.name = 'SkillIndexerError';
+  }
+}
+
+class GitHubError extends SkillIndexerError {
+  constructor(message: string, public statusCode?: number, cause?: Error) {
+    super(message, 'GITHUB_ERROR', cause);
+  }
+}
+
+class ParseError extends SkillIndexerError {
+  constructor(message: string, public filePath: string, cause?: Error) {
+    super(message, 'PARSE_ERROR', cause);
+  }
+}
+
+class ClassificationError extends SkillIndexerError {
+  constructor(message: string, public skillId: string, cause?: Error) {
+    super(message, 'CLASSIFICATION_ERROR', cause);
+  }
+}
+
+class DatabaseError extends SkillIndexerError {
+  constructor(message: string, cause?: Error) {
+    super(message, 'DATABASE_ERROR', cause);
+  }
+}
+```
+
+### Retry Logic
+
+```typescript
+async function withRetry<T>(
+  fn: () => Promise<T>,
+  options: {
+    maxRetries: number;
+    backoffMs: number;
+    backoffMultiplier: number;
+    shouldRetry?: (error: Error) => boolean;
+  }
+): Promise<T> {
+  let lastError: Error;
+
+  for (let attempt = 0; attempt <= options.maxRetries; attempt++) {
+    try {
+      return await fn();
+    } catch (error) {
+      lastError = error as Error;
+
+      if (options.shouldRetry && !options.shouldRetry(lastError)) {
+        throw lastError;
+      }
+
+      if (attempt < options.maxRetries) {
+        const delay = options.backoffMs * Math.pow(options.backoffMultiplier, attempt);
+        await sleep(delay);
+      }
+    }
+  }
+
+  throw lastError!;
+}
+```
+
+## PostgreSQL Migration Path
+
+When scaling beyond SQLite, the migration path involves:
+
+1. **Schema Translation**: Drizzle ORM supports both SQLite and PostgreSQL with minimal schema changes
+
+2. **Vector Search**: Add pgvector for semantic search
+   ```sql
+   CREATE EXTENSION vector;
+   ALTER TABLE skills ADD COLUMN embedding vector(1536);
+   CREATE INDEX ON skills USING ivfflat (embedding vector_cosine_ops);
+   ```
+
+3. **Connection Pooling**: Use connection pool for concurrent access
+   ```typescript
+   import { Pool } from 'pg';
+   import { drizzle } from 'drizzle-orm/node-postgres';
+   ```
+
+4. **Data Migration**: Export from SQLite, import to PostgreSQL
+   ```bash
+   skillindexer export --format sql > backup.sql
+   psql skillindexer < backup.sql
+   ```
+
+## Testing Strategy
+
+### Unit Tests
+- Parser tests with fixture SKILL.md files
+- Database adapter tests with in-memory SQLite
+- Classification prompt tests with mocked responses
+
+### Integration Tests
+- End-to-end scraping with real GitHub API (rate-limited)
+- Classification with real Claude API (cost-limited)
+- Full CLI command tests
+
+### Fixtures
+```
+test/fixtures/
+├── skills/
+│   ├── valid-skill/
+│   │   ├── SKILL.md
+│   │   └── _meta.json
+│   ├── minimal-skill/
+│   │   └── SKILL.md
+│   └── invalid-skill/
+│       └── SKILL.md
+├── awesome-lists/
+│   └── README.md
+└── api-responses/
+    ├── github/
+    └── claude/
+```
+
+## Security Considerations
+
+1. **API Keys**: Never log or expose API keys; use environment variables
+2. **Input Validation**: Validate all parsed content before database insertion
+3. **SQL Injection**: Use parameterized queries (Drizzle ORM handles this)
+4. **Rate Limiting**: Respect GitHub and Anthropic rate limits
+5. **Cache Poisoning**: Validate cached content before use