glost 0.4.0 → 0.5.0

package/package.json

@@ -1,59 +1,44 @@
 {
   "name": "glost",
-  "version": "0.4.0",
-  "description": "GLOST - Glossed Syntax Tree for augmenting text with language learning metadata",
+  "version": "0.5.0",
+  "description": "GLOST - Glossed Syntax Tree for language learning and multilingual text processing",
   "author": "fustilio",
   "license": "MIT",
   "repository": {
     "type": "git",
     "url": "https://github.com/fustilio/glost.git",
-    "directory": "packages/core"
+    "directory": "packages/glost"
   },
   "type": "module",
-  "main": "./src/index.ts",
-  "types": "./src/index.ts",
-  "bin": {
-    "glost": "./dist/cli/migrate.js"
-  },
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
   "exports": {
     ".": {
-      "types": "./src/index.ts",
-      "default": "./src/index.ts"
-    },
-    "./nodes": {
-      "types": "./src/nodes.ts",
-      "default": "./src/nodes.ts"
+      "types": "./dist/index.d.ts",
+      "default": "./dist/index.js"
     },
-    "./utils": {
-      "types": "./src/utils.ts",
-      "default": "./src/utils.ts"
+    "./processor": {
+      "types": "./dist/processor.d.ts",
+      "default": "./dist/processor.js"
     },
-    "./validators": {
-      "types": "./src/validators.ts",
-      "default": "./src/validators.ts"
+    "./registry": {
+      "types": "./dist/registry.d.ts",
+      "default": "./dist/registry.js"
     },
-    "./guards": {
-      "types": "./src/guards.ts",
-      "default": "./src/guards.ts"
+    "./presets": {
+      "types": "./dist/presets.d.ts",
+      "default": "./dist/presets.js"
     }
   },
   "dependencies": {
-    "unist-util-filter": "^5.0.1",
-    "unist-util-find": "^3.0.0",
-    "unist-util-flatmap": "^1.0.0",
-    "unist-util-is": "^6.0.0",
-    "unist-util-map": "^4.0.0",
-    "unist-util-modify-children": "^4.0.0",
-    "unist-util-remove": "^4.0.0",
-    "unist-util-select": "^5.1.0",
-    "unist-util-visit": "^5.0.0",
-    "zod": "^3.23.8",
-    "glost-common": "0.2.0"
+    "glost-core": "0.5.0",
+    "glost-processor": "0.5.0",
+    "glost-presets": "0.5.0",
+    "glost-registry": "0.5.0"
   },
   "devDependencies": {
-    "@types/nlcst": "^2.0.3",
-    "typescript": "^5.8.3",
-    "vitest": "^3.0.5"
+    "@types/node": "^20.0.0",
+    "typescript": "^5.3.3"
   },
   "keywords": [
     "glost",
@@ -63,11 +48,13 @@
     "language-learning",
     "linguistics",
     "syntax-tree",
-    "unist"
+    "unist",
+    "processor",
+    "pipeline",
+    "unified"
   ],
   "scripts": {
     "build": "tsc",
-    "typecheck": "tsc --noEmit",
-    "test": "vitest --passWithNoTests"
+    "typecheck": "tsc --noEmit"
   }
 }

package/src/index.ts

@@ -1,69 +1,126 @@
-// GLOST - Glossed Syntax Tree
-// Extends nlcst for language learning with annotations
-
-export * from "./types.js";
-export * from "./nodes.js";
-export * from "./utils.js";
-export * from "./validators.js";
-export * from "./guards.js";
-export * from "./errors.js";
-// export * from './example.js';
-
-// Re-export utility types
-export type { ParagraphLike } from "./utils.js";
-
-// Re-export key utilities for transcription components
-export {
-  // Tree traversal
-  getAllWords,
-  getAllSentences,
-  getAllParagraphs,
-  getAllClauses,
-  getAllPhrases,
-  getAllSyllables,
-  getAllCharacters,
-  getWordsFromDocument,
-  getFirstSentence,
-  getWordsFromSentence,
-  getWordsFromParagraph,
-
-  // Node finding
-  findNodesByType,
-  findWordsByLanguage,
-  findWordsByTranscriptionSystem,
-
-  // Type guards (most are now exported via guards.ts)
-  isGLOSTWord,
-  isGLOSTSentence,
-  isGLOSTParagraph,
-  isGLOSTRoot,
-  isGLOSTClause,
-  isGLOSTPhrase,
-  isGLOSTSyllable,
-  isGLOSTCharacter,
-
-  // Word utilities
-  getWordText,
-  getWordTranscription,
-  hasWordTranscription,
-  getWordTranslation,
-  getWordMeaning,
-  getWordPartOfSpeech,
-  getWordDifficulty,
-
-  // Sentence utilities
-  getSentenceTranslation,
-
-  // Content statistics utilities
-  getGLOSTWordCount,
-  adaptParagraphLikeToGLOST,
-
-  // BCP-47 Language utilities
-  parseLanguageTag,
-  getBaseLanguage,
-  areLanguagesCompatible,
-  findBestLanguageMatch,
-  getLanguageFallback,
-  normalizeLanguageTag,
-  isValidLanguageTag,
-} from "./utils.js";
+/**
+ * GLOST - Glossed Syntax Tree
+ * 
+ * Main package that re-exports all GLOST functionality.
+ * 
+ * @packageDocumentation
+ * 
+ * @example
+ * ```typescript
+ * // Import everything from one place
+ * import { glost, createSimpleDocument, getAllWords } from "glost";
+ * import { languageLearningPreset } from "glost/presets";
+ * 
+ * // Create a document
+ * const document = createSimpleDocument(words, "th", "thai");
+ * 
+ * // Process with plugins
+ * const result = await glost()
+ *   .use(languageLearningPreset)
+ *   .process(document);
+ * ```
+ */
+
+// ============================================================================
+// Core Types
+// ============================================================================
+export type {
+  GLOSTNode,
+  GLOSTRoot,
+  GLOSTParagraph,
+  GLOSTSentence,
+  GLOSTWord,
+  GLOSTText,
+  GLOSTWhiteSpace,
+  GLOSTPunctuation,
+  GLOSTSymbol,
+  GLOSTSource,
+  GLOSTClause,
+  GLOSTPhrase,
+  GLOSTSyllable,
+  GLOSTCharacter,
+  GLOSTExtras,
+  TransliterationData,
+  LinguisticMetadata,
+  LanguageCode,
+  ScriptSystem,
+} from "glost-core";
+
+// ============================================================================
+// Node Factories
+// ============================================================================
+export {
+  createGLOSTRootNode,
+  createGLOSTParagraphNode,
+  createGLOSTSentenceNode,
+  createGLOSTWordNode,
+  createGLOSTTextNode,
+  createGLOSTWhiteSpaceNode,
+  createGLOSTPunctuationNode,
+  createGLOSTSymbolNode,
+  createSimpleDocument,
+  createDocumentFromSentences,
+  createDocumentFromParagraphs,
+  createSentenceFromWords,
+  createParagraphFromSentences,
+  createSimpleWord,
+  NODE_TYPES,
+} from "glost-core";
+
+// ============================================================================
+// Tree Utilities
+// ============================================================================
+export {
+  getAllWords,
+  getAllSentences,
+  getAllParagraphs,
+  getAllClauses,
+  getFirstSentence,
+  getWordsFromSentence,
+  getWordsFromParagraph,
+  findWordsByLanguage,
+  isGLOSTWord,
+  isGLOSTSentence,
+  isGLOSTParagraph,
+  isGLOSTRoot,
+  getWordText,
+  getWordTranscription,
+  getWordTranslation,
+  getSentenceTranslation,
+  getGLOSTWordCount,
+} from "glost-core";
+
+// ============================================================================
+// Processor API
+// ============================================================================
+export { glost, GLOSTProcessor } from "glost-processor";
+export type {
+  FrozenProcessor,
+  Plugin,
+  PluginSpec,
+  Preset,
+  ProcessorOptions,
+  ProcessingResult,
+  ProcessingError,
+  ProcessingWarning,
+  ProcessingStats,
+  BeforeHook,
+  AfterHook,
+  ErrorHook,
+  SkipHook,
+  ProgressHook,
+  ProgressStats,
+} from "glost-processor";
+
+// ============================================================================
+// Plugin Registry
+// ============================================================================
+export { pluginRegistry, PluginRegistry } from "glost-registry";
+export type {
+  PluginMetadata,
+  PluginCategory,
+  PluginCapabilities,
+  PluginQuery,
+  ConflictReport,
+  ValidationResult,
+} from "glost-registry";

package/src/presets.ts

@@ -0,0 +1,18 @@
+/**
+ * Presets exports
+ * 
+ * Re-exports preset configurations for convenient access via glost/presets.
+ */
+
+export {
+  languageLearningPreset,
+  createLanguageLearningPreset,
+  readingAppPreset,
+  createReadingAppPreset,
+  vocabularyBuilderPreset,
+  createVocabularyBuilderPreset,
+  grammarAnalyzerPreset,
+  createGrammarAnalyzerPreset,
+  minimalPreset,
+  createMinimalPreset,
+} from "glost-presets";

package/src/processor.ts

@@ -0,0 +1,24 @@
+/**
+ * Processor exports
+ * 
+ * Re-exports the processor API for convenient access via glost/processor.
+ */
+
+export { glost, GLOSTProcessor } from "glost-processor";
+export type {
+  FrozenProcessor,
+  Plugin,
+  PluginSpec,
+  Preset,
+  ProcessorOptions,
+  ProcessingResult,
+  ProcessingError,
+  ProcessingWarning,
+  ProcessingStats,
+  BeforeHook,
+  AfterHook,
+  ErrorHook,
+  SkipHook,
+  ProgressHook,
+  ProgressStats,
+} from "glost-processor";

package/src/registry.ts

@@ -0,0 +1,23 @@
+/**
+ * Registry exports
+ * 
+ * Re-exports the registry API for convenient access via glost/registry.
+ */
+
+export { pluginRegistry, PluginRegistry, PluginFilter, PluginSorter, PluginDiscovery, PluginValidator } from "glost-registry";
+export type {
+  PluginMetadata,
+  PluginCategory,
+  PluginCapabilities,
+  PluginRequirements,
+  PluginOptionsSchema,
+  PropertySchema,
+  PluginExample,
+  PluginQuery,
+  ConflictReport,
+  PluginConflict,
+  ValidationResult,
+  ValidationError,
+  ValidationWarning,
+  RegistryStatistics,
+} from "glost-registry";

package/tsconfig.json

@@ -1,9 +1,12 @@
 {
-  "extends": "../../tsconfig.json",
+  "extends": "../../tsconfig.base.json",
   "compilerOptions": {
+    "outDir": "./dist",
     "rootDir": "./src",
-    "outDir": "./dist"
+    "declaration": true,
+    "declarationMap": true,
+    "sourceMap": true
   },
   "include": ["src/**/*"],
-  "exclude": ["node_modules", "dist", "**/*.test.ts", "**/*.spec.ts", "src/__tests__"]
+  "exclude": ["node_modules", "dist"]
 }

package/CHANGELOG.md

@@ -1,296 +0,0 @@
-# glost
-
-## 0.4.0
-
-### Minor Changes
-
-- Fixed critical production issues (ESM imports, BCP-47 language standardization, removed redundant transcription schema fields, renamed translation API sourceLanguage/targetLanguage → from/to) and added major DX improvements (typed extras via declaration merging, comprehensive error classes with suggestions, standard GLOSTDataProvider interface, built-in migration CLI tool npx glost migrate).
-
-### Patch Changes
-
-- Updated dependencies
-  - glost-common@0.2.0
-  - glost-utils@0.2.0
-
-## 0.4.0
-
-### Major Changes
-
-GLOST v0.4.0 brings critical production fixes and developer experience improvements based on 6+ months of real-world usage feedback from the lalia-prism team processing 50+ language learning stories across Thai and French.
-
-#### Breaking Changes
-
-**1. ESM Imports Fixed** 🔧
-
-- Added `.js` extensions to all barrel exports for proper Node.js ESM support
-- Updated TypeScript config to `moduleResolution: "node16"`
-- **Impact**: Fixes `Cannot find module` errors in Node.js ESM
-- **Migration**: Automatic when upgrading packages
-
-**2. BCP-47 Language Code Standard** 🌍
-
-- All language codes now standardized on BCP-47 format (e.g., `en-US`, `th-TH`)
-- Added comprehensive language utilities: `normalizeLanguageCode()`, `matchLanguage()`, `parseLanguageCode()`, `findBestMatch()`, `getLanguageFallbacks()`
-- **Impact**: Language codes like `"th"` become `"th-TH"`
-- **Migration**: Use `npx glost-migrate v0.3-to-v0.4 ./src` or `migrateAllLanguageCodes()` utility
-
-**3. Transcription Schema Cleanup** ✨
-
-- Removed redundant `system` field from `TranscriptionInfo` type
-- System name is already the key in the transcription object
-- **Before**: `{ ipa: { text: "hello", system: "ipa" } }`
-- **After**: `{ ipa: { text: "hello" } }`
-- **Migration**: Use `migrateTranscriptionSchema()` utility
-
-**4. Translation API Clarity** 📝
-
-- Renamed confusing `sourceLanguage`/`targetLanguage` to clear `from`/`to`
-- **Before**: `createTranslationExtension({ sourceLanguage: "th", targetLanguage: "en", ... })`
-- **After**: `createTranslationExtension({ from: "th", to: "en", ... })`
-- **Migration**: Manual update of extension calls
-
-#### New Features
-
-**Typed Extras via Declaration Merging** 🎯
-
-- Extensions can now augment `GLOSTExtras` interface for full type safety
-- Automatic TypeScript autocomplete for extension fields
-- No runtime overhead, pure type-level enhancement
-
-```typescript
-// In extension package
-declare module "glost" {
-  interface GLOSTExtras {
-    frequency?: {
-      rank: number;
-      category: "very-common" | "common" | "uncommon" | "rare";
-    };
-  }
-}
-
-// In user code - full autocomplete!
-word.extras.frequency?.rank; // number | undefined
-```
-
-**Standard Provider Interface** 🔌
-
-- New `GLOSTDataProvider<TInput, TOutput>` interface for consistency
-- Built-in support for batch processing, caching, and cleanup
-- Helper utilities: `createSimpleProvider()`, `createCachedProvider()`, `createFallbackProvider()`
-
-**Better Error Messages** 💬
-
-- New error classes with context, suggestions, and documentation links
-- Helpful error formatting with file paths and node locations
-- Error types: `GLOSTValidationError`, `GLOSTExtensionError`, `GLOSTProviderError`, etc.
-
-```typescript
-GLOSTValidationError: Missing required field 'text' on WordNode
-
-  Location: document.children[0]
-  Node type: WordNode
-  File: stories/test.glost.json:42
-
-  Suggestion: Add a 'text' field containing the word's text content.
-  Documentation: https://glost.dev/docs/node-types#wordnode
-```
-
-**Language Code Utilities** 🛠️
-
-- `normalizeLanguageCode()` - Convert to BCP-47
-- `matchLanguage()` - Flexible matching with options
-- `parseLanguageCode()` - Parse into components
-- `findBestMatch()` - Find closest match from available codes
-- `getLanguageFallbacks()` - Get fallback chain
-- `isValidBCP47()` - Validation
-- `asBCP47()` - Type-safe casting
-
-**Migration CLI** 🚀
-
-- Built-in migration CLI: `npx glost migrate`
-- Automated migration from v0.3.x to v0.4.0
-- Commands: `v0.3-to-v0.4`, `analyze`, `help`
-- Dry-run support for safe testing
-- Programmatic migration utilities in `glost-utils`
-
-### Improvements
-
-- **Bundle Size**: Foundation laid for future optimizations
-- **Type Safety**: Changed `GLOSTExtras` from type to interface for declaration merging
-- **Documentation**: Comprehensive migration guide and updated language docs
-- **Developer Experience**: Better error messages save significant debugging time
-
-### Fixes
-
-- Fixed ESM imports in all language packages
-- Fixed TypeScript moduleResolution for proper ESM output
-- Removed redundant schema fields reducing JSON size
-
-### Migration Guide
-
-See [MIGRATION_v0.3_to_v0.4.md](../../MIGRATION_v0.3_to_v0.4.md) for complete upgrade instructions.
-
-Quick migration:
-
-```bash
-npx glost migrate v0.3-to-v0.4 ./src
-```
-
-### Acknowledgments
-
-Special thanks to the lalia-prism team for their comprehensive RFC and 6+ months of production feedback that made this release possible. Their real-world experience with Thai and French language learning content identified critical issues and guided these improvements.
-
-### Dependencies
-
-- glost-common@0.4.0
-- glost-utils@0.4.0
-
-## 0.3.0
-
-### Minor Changes
-
-- Externalized language-specific helpers and data-dependent extensions into dedicated packages. Language helpers (`createThaiWord`, `createJapaneseWord`) moved to `glost-th` and `glost-ja`. Extensions (frequency, difficulty, POS, gender, clause-segmenter, transcription, translation) now require explicit data providers instead of fallback data. See `MIGRATION_EXTENSIONS.md` for migration guide.
-
-### Patch Changes
-
-- Updated dependencies
-  - glost-common@0.1.3
-
-## 0.2.0
-
-### Minor Changes
-
-- # Externalize Language-Specific Helpers to Dedicated Packages
-
-  ## Breaking Changes
-
-  Language-specific helper functions have been moved from the core `glost` package into dedicated language packages (`glost-th`, `glost-ja`). This keeps the core lightweight, reduces bundle sizes, and allows for independent language support.
-
-  ### Removed from `glost` (Breaking)
-  - `createThaiWord()` → moved to `glost-th` package
-  - `createJapaneseWord()` → moved to `glost-ja` package
-  - `CreateThaiWordOptions` interface → moved to `glost-th`
-  - `CreateJapaneseWordOptions` interface → moved to `glost-ja`
-
-  ### Migration Required
-
-  Install language packages separately:
-
-  ```bash
-  npm install glost-th  # Thai language support
-  npm install glost-ja  # Japanese language support
-  ```
-
-  Update imports in your code:
-
-  ```typescript
-  // Before (v0.1.x)
-  import { createThaiWord, createJapaneseWord } from "glost";
-
-  // After (v0.2.0+)
-  import { createThaiWord } from "glost-th";
-  import { createJapaneseWord } from "glost-ja";
-  ```
-
-  See [MIGRATION.md](../../MIGRATION.md) for complete upgrade guide.
-
-  ## New Packages
-
-  ### `glost-th` - Thai Language Support
-
-  Initial release providing:
-  - `createThaiWord()` helper
-  - Thai transcription provider interfaces
-  - Support for RTGS, Paiboon, Paiboon+, AUA, and IPA schemes
-
-  ### `glost-ja` - Japanese Language Support
-
-  Initial release providing:
-  - `createJapaneseWord()` helper
-  - Japanese transcription provider interfaces
-  - Support for Romaji (Hepburn, Kunrei, Nihon), Furigana, and IPA schemes
-
-  ## Internal Improvements
-  - Fixed circular dependencies in core package (changed `glost/src/*` imports to relative imports)
-  - Reduced core package size by ~30%
-  - Improved build performance
-  - Better modularity and maintainability
-
-  ## What Stays in Core
-
-  All core functionality remains unchanged:
-  - ✅ `createGLOSTWordNode()` - Generic word creation
-  - ✅ `createGLOSTSentenceNode()` - Sentence creation
-  - ✅ `createGLOSTRootNode()` - Document creation
-  - ✅ `createSimpleWord()` - Simple word helper
-  - ✅ All tree traversal utilities
-  - ✅ Type guards and validators
-  - ✅ All core types and interfaces
-
-  ## Benefits
-  - **Smaller bundles** - Import only languages you use
-  - **Independent releases** - Update languages without core changes
-  - **Better organization** - All Thai code in `glost-th`, all Japanese in `glost-ja`
-  - **Easier contributions** - Add new languages without touching core
-  - **Scalability** - Can support 50+ languages without bloating core
-
-### Patch Changes
-
-- Updated dependencies
-  - glost-common@0.1.2
-
-## 0.2.0
-
-### Breaking Changes
-
-- **BREAKING:** Removed `createThaiWord()` - moved to `glost-th` package
-- **BREAKING:** Removed `createJapaneseWord()` - moved to `glost-ja` package
-- **BREAKING:** Removed `CreateThaiWordOptions` interface - moved to `glost-th`
-- **BREAKING:** Removed `CreateJapaneseWordOptions` interface - moved to `glost-ja`
-
-### Migration
-
-Install language packages separately:
-
-```bash
-npm install glost-th  # Thai language support
-npm install glost-ja  # Japanese language support
-```
-
-Update imports:
-
-```typescript
-// Before (v0.1.x)
-import { createThaiWord, createJapaneseWord } from "glost";
-
-// After (v0.2.0+)
-import { createThaiWord } from "glost-th";
-import { createJapaneseWord } from "glost-ja";
-```
-
-See [MIGRATION.md](../../MIGRATION.md) for complete upgrade guide.
-
-### Internal Changes
-
-- Fixed circular dependencies (changed `glost/src/*` imports to relative imports)
-- Reduced package size by ~30%
-- Improved build performance
-
-### What Stays in Core
-
-All core functionality remains unchanged:
-
-- ✅ `createGLOSTWordNode()` - Generic word creation
-- ✅ `createGLOSTSentenceNode()` - Sentence creation
-- ✅ `createGLOSTRootNode()` - Document creation
-- ✅ `createSimpleWord()` - Simple word helper
-- ✅ All tree traversal utilities
-- ✅ Type guards and validators
-
-## 0.1.1
-
-### Patch Changes
-
-- fix names
-- Updated dependencies
-  - glost-common@0.1.1

package/dist/cli/migrate.d.ts

@@ -1,8 +0,0 @@
-#!/usr/bin/env node
-/**
- * GLOST Migration CLI
- *
- * Command-line interface for migrating GLOST documents
- */
-export {};
-//# sourceMappingURL=migrate.d.ts.map

package/dist/cli/migrate.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"migrate.d.ts","sourceRoot":"","sources":["../../src/cli/migrate.ts"],"names":[],"mappings":";AACA;;;;GAIG"}