@sylphx/flow 1.0.1 → 1.0.3

package/src/types.ts

@@ -0,0 +1,40 @@
+/**
+ * Centralized type exports
+ * This file re-exports types from domain-specific modules for backward compatibility
+ *
+ * MIGRATION GUIDE:
+ * Instead of importing from this file, prefer importing from specific type modules:
+ * - './types/cli.types.js' for CLI-related types
+ * - './types/mcp.types.js' for MCP configuration types
+ * - './types/target.types.js' for Target system types
+ * - './types/common.types.js' for shared types
+ */
+
+// Re-export all types for backward compatibility
+export type {
+  CommandHandler,
+  CommandOptions,
+  CommandConfig,
+  RunCommandOptions,
+} from './types/cli.types.js';
+export type {
+  CommonOptions,
+  Resolvable,
+  SetupResult,
+} from './types/common.types.js';
+export type {
+  MCPServerConfig,
+  MCPServerConfigFlags,
+  MCPServerConfigHTTP,
+  MCPServerConfigUnion,
+  OpenCodeConfig,
+} from './types/mcp.types.js';
+export {
+  isCLICommandConfig,
+  isHttpConfig,
+  isStdioConfig,
+} from './types/mcp.types.js';
+export type {
+  Target,
+  TargetConfig,
+} from './types/target.types.js';

package/src/utils/advanced-tokenizer.ts

@@ -0,0 +1,191 @@
+/**
+ * Advanced Code Tokenizer - 純粹 StarCoder2，無任何多餘處理
+ * 完全信任 StarCoder2 嘅世界級 tokenization 能力
+ */
+
+import { AutoTokenizer } from '@huggingface/transformers';
+
+export interface AdvancedToken {
+  text: string;
+  id: number;
+  score: number;
+  confidence: number;
+  relevance: 'high' | 'medium' | 'low';
+}
+
+export interface AdvancedTokenizerResult {
+  tokens: AdvancedToken[];
+  metadata: {
+    totalTokens: number;
+    vocabSize: number;
+    processingTime: number;
+    averageConfidence: number;
+  };
+  raw: {
+    inputIds: number[];
+    decodedText: string;
+  };
+}
+
+/**
+ * Advanced Code Tokenizer - 純粹 StarCoder2
+ */
+export class AdvancedCodeTokenizer {
+  private tokenizer: any;
+  private initialized = false;
+  private modelPath: string;
+
+  constructor(
+    options: {
+      modelPath?: string;
+    } = {}
+  ) {
+    this.modelPath = options.modelPath || './models/starcoder2';
+  }
+
+  /**
+   * 初始化 tokenizer
+   */
+  async initialize(): Promise<void> {
+    if (this.initialized) {
+      return;
+    }
+
+    try {
+      this.tokenizer = await AutoTokenizer.from_pretrained(this.modelPath);
+      this.initialized = true;
+    } catch (error) {
+      throw new Error(`Tokenizer initialization failed: ${error.message}`);
+    }
+  }
+
+  /**
+   * 純粹 StarCoder2 tokenization - 無任何安全限制或多餘分析
+   */
+  async tokenize(content: string): Promise<AdvancedTokenizerResult> {
+    if (!this.initialized) {
+      await this.initialize();
+    }
+
+    const startTime = Date.now();
+
+    // Handle empty content
+    if (!content || content.trim().length === 0) {
+      return {
+        tokens: [],
+        metadata: {
+          totalTokens: 0,
+          vocabSize: 49152,
+          processingTime: Date.now() - startTime,
+          averageConfidence: 0,
+        },
+        raw: {
+          inputIds: [],
+          decodedText: '',
+        },
+      };
+    }
+
+    try {
+      // 完全信任 StarCoder2，直接處理所有內容
+      const encoded = await this.tokenizer(content);
+      const inputIds = encoded.input_ids.tolist()[0];
+
+      // 解碼獲得原文
+      const decodedText = await this.tokenizer.decode(inputIds);
+
+      // 直接用 StarCoder2 嘅輸出，唔做多餘分析
+      const tokens = await this.createDirectTokens(decodedText, inputIds);
+
+      const processingTime = Date.now() - startTime;
+
+      return {
+        tokens,
+        metadata: {
+          totalTokens: tokens.length,
+          vocabSize: 49152,
+          processingTime,
+          averageConfidence: 0.95, // StarCoder2 本身就係高質量
+        },
+        raw: {
+          inputIds,
+          decodedText,
+        },
+      };
+    } catch (error) {
+      throw new Error(`Tokenization failed: ${error.message}`);
+    }
+  }
+
+  /**
+   * 純粹 StarCoder2 tokenization - 完全信任，無任何額外處理
+   * 直接使用 StarCoder2 嘅 token IDs，逐個解碼成文字
+   */
+  private async createDirectTokens(
+    _decodedText: string,
+    inputIds: number[]
+  ): Promise<AdvancedToken[]> {
+    const tokens: AdvancedToken[] = [];
+
+    // 完全信任 StarCoder2，直接使用佢嘅 tokenization
+    // 逐個 token ID 解碼，得到 StarCoder2 認為嘅最佳分割
+    for (let i = 0; i < inputIds.length; i++) {
+      const tokenId = inputIds[i];
+      try {
+        // 直接使用 StarCoder2 嘅解碼結果
+        const tokenText = await this.tokenizer.decode([tokenId], { skip_special_tokens: true });
+        const cleaned = tokenText.trim().toLowerCase();
+
+        // 只過濾空白 token，其他全部保留（完全信任 StarCoder2）
+        if (cleaned.length > 0) {
+          tokens.push({
+            text: cleaned,
+            id: tokenId,
+            score: 1.0, // StarCoder2 嘅所有 token 都係高質量
+            confidence: 1.0, // 完全信任 StarCoder2
+            relevance: 'high' as const,
+          });
+        }
+      } catch (_error) {}
+    }
+
+    return tokens;
+  }
+
+  /**
+   * 便利方法：只返回高質量 tokens
+   */
+  async getTopTokens(content: string, limit = 20): Promise<AdvancedToken[]> {
+    const result = await this.tokenize(content);
+    return result.tokens.slice(0, limit);
+  }
+
+  /**
+   * 便利方法：返回所有 tokens (StarCoder2 嘅輸出全部都係高質量)
+   */
+  async getTechnicalTokens(content: string): Promise<AdvancedToken[]> {
+    const result = await this.tokenize(content);
+    return result.tokens; // StarCoder2 嘅所有輸出都係技術相關
+  }
+
+  /**
+   * 解碼 token IDs 回文本
+   */
+  async decode(tokenIds: number[]): Promise<string> {
+    if (!this.initialized) {
+      throw new Error('Tokenizer not initialized. Call initialize() first.');
+    }
+    return await this.tokenizer.decode(tokenIds);
+  }
+
+  /**
+   * 編碼文本為 token IDs
+   */
+  async encode(text: string): Promise<number[]> {
+    if (!this.initialized) {
+      throw new Error('Tokenizer not initialized. Call initialize() first.');
+    }
+    const result = await this.tokenizer(text);
+    return result.input_ids.tolist()[0];
+  }
+}

package/src/utils/agent-enhancer.ts

@@ -0,0 +1,114 @@
+/**
+ * Agent Enhancer - Append rules and output styles to agent content
+ *
+ * This module provides utilities to enhance agent files with:
+ * - Rules (from assets/rules/core.md)
+ * - Output Styles (from assets/output-styles/*.md)
+ *
+ * These are appended to agent content to ensure every agent has
+ * access to the same rules and output styles without duplicating
+ * them in CLAUDE.md or other system prompts.
+ */
+
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import { getOutputStylesDir, getRulesDir } from './paths.js';
+import { yamlUtils } from './target-utils.js';
+
+/**
+ * Load and combine rules and output styles
+ */
+export async function loadRulesAndStyles(ruleNames?: string[]): Promise<string> {
+  const sections: string[] = [];
+
+  // Load rules (either specified rules or default to 'core')
+  const rulesContent = await loadRules(ruleNames);
+  if (rulesContent) {
+    sections.push(rulesContent);
+  }
+
+  // Load output styles
+  const stylesContent = await loadOutputStyles();
+  if (stylesContent) {
+    sections.push(stylesContent);
+  }
+
+  return sections.join('\n\n---\n\n');
+}
+
+/**
+ * Load rules from assets/rules/*.md
+ * @param ruleNames - Array of rule file names (without .md extension). Defaults to ['core']
+ */
+async function loadRules(ruleNames?: string[]): Promise<string> {
+  try {
+    const rulesDir = getRulesDir();
+    const rulesToLoad = ruleNames && ruleNames.length > 0 ? ruleNames : ['core'];
+    const sections: string[] = [];
+
+    for (const ruleName of rulesToLoad) {
+      const rulePath = path.join(rulesDir, `${ruleName}.md`);
+
+      try {
+        const content = await fs.readFile(rulePath, 'utf8');
+        // Strip YAML front matter
+        const stripped = await yamlUtils.stripFrontMatter(content);
+        sections.push(stripped);
+      } catch (error) {
+        // Log warning if rule file not found, but continue with other rules
+        console.warn(`Warning: Rule file not found: ${ruleName}.md`);
+      }
+    }
+
+    return sections.join('\n\n---\n\n');
+  } catch (_error) {
+    // If rules directory doesn't exist, return empty string
+    return '';
+  }
+}
+
+/**
+ * Load output styles from assets/output-styles/
+ */
+async function loadOutputStyles(): Promise<string> {
+  try {
+    const outputStylesDir = getOutputStylesDir();
+    const files = await fs.readdir(outputStylesDir);
+    const mdFiles = files.filter((f) => f.endsWith('.md'));
+
+    if (mdFiles.length === 0) {
+      return '';
+    }
+
+    const sections: string[] = [];
+
+    for (const file of mdFiles) {
+      const filePath = path.join(outputStylesDir, file);
+      const content = await fs.readFile(filePath, 'utf8');
+
+      // Strip YAML front matter
+      const stripped = await yamlUtils.stripFrontMatter(content);
+      sections.push(stripped);
+    }
+
+    return sections.join('\n\n');
+  } catch (_error) {
+    // If output styles directory doesn't exist, return empty string
+    return '';
+  }
+}
+
+/**
+ * Enhance agent content by appending rules and output styles
+ * @param agentContent - The agent markdown content
+ * @param ruleNames - Optional array of rule file names to include (defaults to ['core'])
+ */
+export async function enhanceAgentContent(agentContent: string, ruleNames?: string[]): Promise<string> {
+  const rulesAndStyles = await loadRulesAndStyles(ruleNames);
+
+  if (!rulesAndStyles) {
+    return agentContent;
+  }
+
+  return `${agentContent}\n\n---\n\n# Rules and Output Styles\n\n${rulesAndStyles}`;
+}

package/src/utils/ai-model-fetcher.ts

@@ -0,0 +1,19 @@
+/**
+ * AI Model Fetcher
+ * Dynamically fetch available models from providers using provider registry
+ */
+
+import type { ProviderId } from '../types/provider.types.js';
+import type { ProviderConfig, ModelInfo } from '../providers/base-provider.js';
+import { getProvider } from '../providers/index.js';
+
+// Re-export ModelInfo for backward compatibility
+export type { ModelInfo } from '../providers/base-provider.js';
+
+/**
+ * Fetch models for a provider using provider registry
+ */
+export async function fetchModels(provider: ProviderId, config: ProviderConfig = {}): Promise<ModelInfo[]> {
+  const providerInstance = getProvider(provider);
+  return providerInstance.fetchModels(config);
+}