kenaz 0.3.2 → 0.3.3

package/core/dev/scripts/matrix/src/cli/commands/stats.ts.wip

@@ -1,168 +1,168 @@
-/**
- * stats command - Display statistics about .matrix file
- */
-
-import chalk from 'chalk';
-import { FileHandler } from '../utils/file-handler';
-import { MatrixParser } from '../../parser';
-import { TagIndexer } from '../../indexer/tag-indexer';
-import type { DataRecord } from '../../../types';
-
-interface StatsOptions {
-  verbose?: boolean;
-}
-
-export async function statsCommand(
-  file: string,
-  options: StatsOptions = {}
-): Promise<void> {
-  const filePath = FileHandler.ensureExtension(file);
-
-  // Read file
-  const content = FileHandler.read(filePath);
-  if (!content) {
-    process.exit(1);
-  }
-
-  // Parse file
-  const parser = new MatrixParser();
-  const result = parser.parse(content);
-
-  if (!result.success) {
-    console.error(chalk.red('Failed to parse file'));
-    process.exit(1);
-  }
-
-  const matrixFile = result.data!;
-
-  // Build tag index
-  const indexer = new TagIndexer();
-  indexer.buildIndex(matrixFile);
-
-  console.log(chalk.bold.cyan('\n📊 Matrix File Statistics\n'));
-  console.log(chalk.gray(`File: ${filePath}\n`));
-
-  // 1. Overview
-  console.log(chalk.bold('Overview:'));
-  console.log(`  Total Rules: ${chalk.green(matrixFile.data.records.length)}`);
-  console.log(`  Version: ${chalk.yellow(matrixFile.meta.v)}`);
-  console.log(`  Category: ${chalk.yellow(matrixFile.meta.cat)}\n`);
-
-  // 2. Priority Distribution
-  const priorityStats = groupBy(matrixFile.data.records, 'p');
-  console.log(chalk.bold('Priority Distribution:'));
-  printDistribution(priorityStats, ['P0', 'P1', 'P2', 'P3']);
-
-  // 3. Source Distribution
-  const sourceStats = groupBy(matrixFile.rules, 'source');
-  console.log(chalk.bold('\nSource Distribution:'));
-  printDistribution(sourceStats);
-
-  // 4. Category Distribution
-  const categoryStats = groupBy(matrixFile.rules, 'category');
-  if (Object.keys(categoryStats).length > 0) {
-    console.log(chalk.bold('\nCategory Distribution:'));
-    printDistribution(categoryStats);
-  }
-
-  // 5. Status Distribution
-  const statusStats = groupBy(matrixFile.rules, 'status');
-  if (Object.keys(statusStats).length > 0) {
-    console.log(chalk.bold('\nStatus Distribution:'));
-    printDistribution(statusStats, [
-      'pending',
-      'in_progress',
-      'completed',
-      'blocked',
-      'cancelled',
-    ]);
-  }
-
-  // 6. Domain Distribution
-  const domainStats = groupBy(matrixFile.rules, 'domain');
-  if (Object.keys(domainStats).length > 0) {
-    console.log(chalk.bold('\nDomain Distribution:'));
-    printDistribution(domainStats);
-  }
-
-  // 7. Tag Statistics
-  const tagStats = indexer.getTagStats();
-  if (tagStats.length > 0) {
-    console.log(chalk.bold('\nTop Tags:'));
-    const topTags = tagStats.slice(0, 10);
-    for (const stat of topTags) {
-      console.log(`  ${chalk.cyan(stat.tag)}: ${chalk.green(stat.count)}`);
-    }
-    if (tagStats.length > 10) {
-      console.log(chalk.gray(`  ... and ${tagStats.length - 10} more`));
-    }
-  }
-
-  // 8. Verbose mode: Show all rules
-  if (options.verbose) {
-    console.log(chalk.bold('\nAll Rules:'));
-    for (const rule of matrixFile.rules) {
-      console.log(`  ${chalk.cyan(rule.id)}: ${rule.title || '(no title)'}`);
-    }
-  }
-
-  console.log(); // Empty line at end
-}
-
-/**
- * Group rules by field
- */
-function groupBy(
-  rules: Rule[],
-  field: keyof Rule
-): Record<string, number> {
-  const result: Record<string, number> = {};
-
-  for (const rule of rules) {
-    const value = rule[field];
-    if (value) {
-      const key = String(value);
-      result[key] = (result[key] || 0) + 1;
-    }
-  }
-
-  return result;
-}
-
-/**
- * Print distribution with bar chart
- */
-function printDistribution(
-  stats: Record<string, number>,
-  order?: string[]
-): void {
-  const entries = Object.entries(stats);
-
-  // Sort by order if provided, otherwise by count descending
-  if (order) {
-    entries.sort((a, b) => {
-      const aIndex = order.indexOf(a[0]);
-      const bIndex = order.indexOf(b[0]);
-      if (aIndex === -1 && bIndex === -1) return 0;
-      if (aIndex === -1) return 1;
-      if (bIndex === -1) return -1;
-      return aIndex - bIndex;
-    });
-  } else {
-    entries.sort((a, b) => b[1] - a[1]);
-  }
-
-  // Calculate max count for bar chart
-  const maxCount = Math.max(...entries.map(([, count]) => count));
-  const maxBarLength = 30;
-
-  for (const [key, count] of entries) {
-    const barLength = Math.ceil((count / maxCount) * maxBarLength);
-    const bar = '█'.repeat(barLength);
-    const percentage = ((count / entries.reduce((sum, [, c]) => sum + c, 0)) * 100).toFixed(1);
-
-    console.log(
-      `  ${chalk.yellow(key.padEnd(15))} ${chalk.green(bar)} ${chalk.cyan(count)} ${chalk.gray(`(${percentage}%)`)}`
-    );
-  }
-}
+/**
+ * stats command - Display statistics about .matrix file
+ */
+
+import chalk from 'chalk';
+import { FileHandler } from '../utils/file-handler';
+import { MatrixParser } from '../../parser';
+import { TagIndexer } from '../../indexer/tag-indexer';
+import type { DataRecord } from '../../../types';
+
+interface StatsOptions {
+  verbose?: boolean;
+}
+
+export async function statsCommand(
+  file: string,
+  options: StatsOptions = {}
+): Promise<void> {
+  const filePath = FileHandler.ensureExtension(file);
+
+  // Read file
+  const content = FileHandler.read(filePath);
+  if (!content) {
+    process.exit(1);
+  }
+
+  // Parse file
+  const parser = new MatrixParser();
+  const result = parser.parse(content);
+
+  if (!result.success) {
+    console.error(chalk.red('Failed to parse file'));
+    process.exit(1);
+  }
+
+  const matrixFile = result.data!;
+
+  // Build tag index
+  const indexer = new TagIndexer();
+  indexer.buildIndex(matrixFile);
+
+  console.log(chalk.bold.cyan('\n📊 Matrix File Statistics\n'));
+  console.log(chalk.gray(`File: ${filePath}\n`));
+
+  // 1. Overview
+  console.log(chalk.bold('Overview:'));
+  console.log(`  Total Rules: ${chalk.green(matrixFile.data.records.length)}`);
+  console.log(`  Version: ${chalk.yellow(matrixFile.meta.v)}`);
+  console.log(`  Category: ${chalk.yellow(matrixFile.meta.cat)}\n`);
+
+  // 2. Priority Distribution
+  const priorityStats = groupBy(matrixFile.data.records, 'p');
+  console.log(chalk.bold('Priority Distribution:'));
+  printDistribution(priorityStats, ['P0', 'P1', 'P2', 'P3']);
+
+  // 3. Source Distribution
+  const sourceStats = groupBy(matrixFile.rules, 'source');
+  console.log(chalk.bold('\nSource Distribution:'));
+  printDistribution(sourceStats);
+
+  // 4. Category Distribution
+  const categoryStats = groupBy(matrixFile.rules, 'category');
+  if (Object.keys(categoryStats).length > 0) {
+    console.log(chalk.bold('\nCategory Distribution:'));
+    printDistribution(categoryStats);
+  }
+
+  // 5. Status Distribution
+  const statusStats = groupBy(matrixFile.rules, 'status');
+  if (Object.keys(statusStats).length > 0) {
+    console.log(chalk.bold('\nStatus Distribution:'));
+    printDistribution(statusStats, [
+      'pending',
+      'in_progress',
+      'completed',
+      'blocked',
+      'cancelled',
+    ]);
+  }
+
+  // 6. Domain Distribution
+  const domainStats = groupBy(matrixFile.rules, 'domain');
+  if (Object.keys(domainStats).length > 0) {
+    console.log(chalk.bold('\nDomain Distribution:'));
+    printDistribution(domainStats);
+  }
+
+  // 7. Tag Statistics
+  const tagStats = indexer.getTagStats();
+  if (tagStats.length > 0) {
+    console.log(chalk.bold('\nTop Tags:'));
+    const topTags = tagStats.slice(0, 10);
+    for (const stat of topTags) {
+      console.log(`  ${chalk.cyan(stat.tag)}: ${chalk.green(stat.count)}`);
+    }
+    if (tagStats.length > 10) {
+      console.log(chalk.gray(`  ... and ${tagStats.length - 10} more`));
+    }
+  }
+
+  // 8. Verbose mode: Show all rules
+  if (options.verbose) {
+    console.log(chalk.bold('\nAll Rules:'));
+    for (const rule of matrixFile.rules) {
+      console.log(`  ${chalk.cyan(rule.id)}: ${rule.title || '(no title)'}`);
+    }
+  }
+
+  console.log(); // Empty line at end
+}
+
+/**
+ * Group rules by field
+ */
+function groupBy(
+  rules: Rule[],
+  field: keyof Rule
+): Record<string, number> {
+  const result: Record<string, number> = {};
+
+  for (const rule of rules) {
+    const value = rule[field];
+    if (value) {
+      const key = String(value);
+      result[key] = (result[key] || 0) + 1;
+    }
+  }
+
+  return result;
+}
+
+/**
+ * Print distribution with bar chart
+ */
+function printDistribution(
+  stats: Record<string, number>,
+  order?: string[]
+): void {
+  const entries = Object.entries(stats);
+
+  // Sort by order if provided, otherwise by count descending
+  if (order) {
+    entries.sort((a, b) => {
+      const aIndex = order.indexOf(a[0]);
+      const bIndex = order.indexOf(b[0]);
+      if (aIndex === -1 && bIndex === -1) return 0;
+      if (aIndex === -1) return 1;
+      if (bIndex === -1) return -1;
+      return aIndex - bIndex;
+    });
+  } else {
+    entries.sort((a, b) => b[1] - a[1]);
+  }
+
+  // Calculate max count for bar chart
+  const maxCount = Math.max(...entries.map(([, count]) => count));
+  const maxBarLength = 30;
+
+  for (const [key, count] of entries) {
+    const barLength = Math.ceil((count / maxCount) * maxBarLength);
+    const bar = '█'.repeat(barLength);
+    const percentage = ((count / entries.reduce((sum, [, c]) => sum + c, 0)) * 100).toFixed(1);
+
+    console.log(
+      `  ${chalk.yellow(key.padEnd(15))} ${chalk.green(bar)} ${chalk.cyan(count)} ${chalk.gray(`(${percentage}%)`)}`
+    );
+  }
+}

package/core/dev/scripts/matrix/src/cli/commands/tokens.ts.wip

@@ -1,126 +1,126 @@
-/**
- * tokens command - Calculate token count and compression ratio
- */
-
-import chalk from 'chalk';
-import * as fs from 'fs';
-import { resolveFilePath } from '../utils/file-handler';
-import { parseMatrixFile } from '../../parser';
-import { serializeMatrixFile } from '../../serializer';
-
-interface TokensOptions {
-  verbose?: boolean;
-}
-
-export async function tokensCommand(
-  file: string,
-  options: TokensOptions = {}
-): Promise<void> {
-  const filePath = resolveFilePath(file);
-
-  // Read original .matrix content
-  const matrixContent = fs.readFileSync(filePath, 'utf-8');
-
-  // Parse and convert to JSON
-  const matrixFile = parseMatrixFile(filePath);
-  const jsonContent = JSON.stringify(matrixFile, null, 2);
-
-  // Calculate token counts (using simple estimation)
-  const matrixTokens = estimateTokenCount(matrixContent);
-  const jsonTokens = estimateTokenCount(jsonContent);
-
-  // Calculate compression ratio
-  const compressionRatio = (jsonTokens / matrixTokens).toFixed(2);
-
-  // Calculate sizes
-  const matrixSize = Buffer.byteLength(matrixContent, 'utf-8');
-  const jsonSize = Buffer.byteLength(jsonContent, 'utf-8');
-  const sizeRatio = (jsonSize / matrixSize).toFixed(2);
-
-  console.log(chalk.bold.cyan('\n🔢 Token Analysis\n'));
-  console.log(chalk.gray(`File: ${filePath}\n`));
-
-  // Token counts
-  console.log(chalk.bold('Token Counts:'));
-  console.log(`  .matrix format:  ${chalk.green(matrixTokens.toLocaleString())} tokens`);
-  console.log(`  JSON format:     ${chalk.yellow(jsonTokens.toLocaleString())} tokens`);
-  console.log(`  Compression:     ${chalk.cyan(compressionRatio)}:1`);
-  console.log();
-
-  // File sizes
-  console.log(chalk.bold('File Sizes:'));
-  console.log(`  .matrix format:  ${chalk.green(formatBytes(matrixSize))}`);
-  console.log(`  JSON format:     ${chalk.yellow(formatBytes(jsonSize))}`);
-  console.log(`  Size ratio:      ${chalk.cyan(sizeRatio)}:1`);
-  console.log();
-
-  // Efficiency analysis
-  console.log(chalk.bold('Efficiency:'));
-  const tokenSavings = jsonTokens - matrixTokens;
-  const tokenSavingsPercent = ((tokenSavings / jsonTokens) * 100).toFixed(1);
-  console.log(`  Token savings:   ${chalk.green(`-${tokenSavings.toLocaleString()}`)} (${tokenSavingsPercent}%)`);
-
-  const sizeSavings = jsonSize - matrixSize;
-  const sizeSavingsPercent = ((sizeSavings / jsonSize) * 100).toFixed(1);
-  console.log(`  Size savings:    ${chalk.green(`-${formatBytes(sizeSavings)}`)} (${sizeSavingsPercent}%)`);
-  console.log();
-
-  // Verbose mode: Show rule breakdown
-  if (options.verbose) {
-    console.log(chalk.bold('Rule Breakdown:'));
-    console.log(`  Total rules:     ${chalk.cyan(matrixFile.rules.length)}`);
-    console.log(`  Avg tokens/rule: ${chalk.cyan(Math.round(matrixTokens / matrixFile.rules.length))}`);
-    console.log();
-  }
-
-  // Compression assessment
-  const compressionQuality = parseFloat(compressionRatio);
-  let assessment = '';
-  let assessmentColor = chalk.green;
-
-  if (compressionQuality >= 5) {
-    assessment = 'Excellent compression! (≥5:1 target met)';
-    assessmentColor = chalk.green.bold;
-  } else if (compressionQuality >= 3) {
-    assessment = 'Good compression (3-5:1)';
-    assessmentColor = chalk.yellow;
-  } else if (compressionQuality >= 2) {
-    assessment = 'Moderate compression (2-3:1)';
-    assessmentColor = chalk.yellow;
-  } else {
-    assessment = 'Low compression (<2:1)';
-    assessmentColor = chalk.red;
-  }
-
-  console.log(assessmentColor(`✓ ${assessment}`));
-  console.log();
-}
-
-/**
- * Estimate token count using simple heuristics
- * (Approximates GPT tokenization without needing tiktoken dependency)
- */
-function estimateTokenCount(text: string): number {
-  // Simple estimation: ~4 characters per token for English text
-  // This is a rough approximation; actual tokenization varies
-  // For more accuracy, use tiktoken library
-
-  // Remove whitespace for better estimation
-  const trimmedText = text.replace(/\s+/g, ' ').trim();
-
-  // Count words and punctuation as separate tokens
-  const words = trimmedText.split(/\s+/).length;
-  const punctuation = (trimmedText.match(/[.,;:!?()[\]{}'"]/g) || []).length;
-
-  // Estimate: words + punctuation + special chars
-  return Math.ceil((words + punctuation) * 1.3);
-}
-
-/**
- * Format bytes to human-readable string
- */
-function formatBytes(bytes: number): string {
-  if (bytes < 1024) return `${bytes} B`;
-  if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(2)} KB`;
-  return `${(bytes / (1024 * 1024)).toFixed(2)} MB`;
-}
+/**
+ * tokens command - Calculate token count and compression ratio
+ */
+
+import chalk from 'chalk';
+import * as fs from 'fs';
+import { resolveFilePath } from '../utils/file-handler';
+import { parseMatrixFile } from '../../parser';
+import { serializeMatrixFile } from '../../serializer';
+
+interface TokensOptions {
+  verbose?: boolean;
+}
+
+export async function tokensCommand(
+  file: string,
+  options: TokensOptions = {}
+): Promise<void> {
+  const filePath = resolveFilePath(file);
+
+  // Read original .matrix content
+  const matrixContent = fs.readFileSync(filePath, 'utf-8');
+
+  // Parse and convert to JSON
+  const matrixFile = parseMatrixFile(filePath);
+  const jsonContent = JSON.stringify(matrixFile, null, 2);
+
+  // Calculate token counts (using simple estimation)
+  const matrixTokens = estimateTokenCount(matrixContent);
+  const jsonTokens = estimateTokenCount(jsonContent);
+
+  // Calculate compression ratio
+  const compressionRatio = (jsonTokens / matrixTokens).toFixed(2);
+
+  // Calculate sizes
+  const matrixSize = Buffer.byteLength(matrixContent, 'utf-8');
+  const jsonSize = Buffer.byteLength(jsonContent, 'utf-8');
+  const sizeRatio = (jsonSize / matrixSize).toFixed(2);
+
+  console.log(chalk.bold.cyan('\n🔢 Token Analysis\n'));
+  console.log(chalk.gray(`File: ${filePath}\n`));
+
+  // Token counts
+  console.log(chalk.bold('Token Counts:'));
+  console.log(`  .matrix format:  ${chalk.green(matrixTokens.toLocaleString())} tokens`);
+  console.log(`  JSON format:     ${chalk.yellow(jsonTokens.toLocaleString())} tokens`);
+  console.log(`  Compression:     ${chalk.cyan(compressionRatio)}:1`);
+  console.log();
+
+  // File sizes
+  console.log(chalk.bold('File Sizes:'));
+  console.log(`  .matrix format:  ${chalk.green(formatBytes(matrixSize))}`);
+  console.log(`  JSON format:     ${chalk.yellow(formatBytes(jsonSize))}`);
+  console.log(`  Size ratio:      ${chalk.cyan(sizeRatio)}:1`);
+  console.log();
+
+  // Efficiency analysis
+  console.log(chalk.bold('Efficiency:'));
+  const tokenSavings = jsonTokens - matrixTokens;
+  const tokenSavingsPercent = ((tokenSavings / jsonTokens) * 100).toFixed(1);
+  console.log(`  Token savings:   ${chalk.green(`-${tokenSavings.toLocaleString()}`)} (${tokenSavingsPercent}%)`);
+
+  const sizeSavings = jsonSize - matrixSize;
+  const sizeSavingsPercent = ((sizeSavings / jsonSize) * 100).toFixed(1);
+  console.log(`  Size savings:    ${chalk.green(`-${formatBytes(sizeSavings)}`)} (${sizeSavingsPercent}%)`);
+  console.log();
+
+  // Verbose mode: Show rule breakdown
+  if (options.verbose) {
+    console.log(chalk.bold('Rule Breakdown:'));
+    console.log(`  Total rules:     ${chalk.cyan(matrixFile.rules.length)}`);
+    console.log(`  Avg tokens/rule: ${chalk.cyan(Math.round(matrixTokens / matrixFile.rules.length))}`);
+    console.log();
+  }
+
+  // Compression assessment
+  const compressionQuality = parseFloat(compressionRatio);
+  let assessment = '';
+  let assessmentColor = chalk.green;
+
+  if (compressionQuality >= 5) {
+    assessment = 'Excellent compression! (≥5:1 target met)';
+    assessmentColor = chalk.green.bold;
+  } else if (compressionQuality >= 3) {
+    assessment = 'Good compression (3-5:1)';
+    assessmentColor = chalk.yellow;
+  } else if (compressionQuality >= 2) {
+    assessment = 'Moderate compression (2-3:1)';
+    assessmentColor = chalk.yellow;
+  } else {
+    assessment = 'Low compression (<2:1)';
+    assessmentColor = chalk.red;
+  }
+
+  console.log(assessmentColor(`✓ ${assessment}`));
+  console.log();
+}
+
+/**
+ * Estimate token count using simple heuristics
+ * (Approximates GPT tokenization without needing tiktoken dependency)
+ */
+function estimateTokenCount(text: string): number {
+  // Simple estimation: ~4 characters per token for English text
+  // This is a rough approximation; actual tokenization varies
+  // For more accuracy, use tiktoken library
+
+  // Remove whitespace for better estimation
+  const trimmedText = text.replace(/\s+/g, ' ').trim();
+
+  // Count words and punctuation as separate tokens
+  const words = trimmedText.split(/\s+/).length;
+  const punctuation = (trimmedText.match(/[.,;:!?()[\]{}'"]/g) || []).length;
+
+  // Estimate: words + punctuation + special chars
+  return Math.ceil((words + punctuation) * 1.3);
+}
+
+/**
+ * Format bytes to human-readable string
+ */
+function formatBytes(bytes: number): string {
+  if (bytes < 1024) return `${bytes} B`;
+  if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(2)} KB`;
+  return `${(bytes / (1024 * 1024)).toFixed(2)} MB`;
+}