embedoc 0.10.1 → 0.11.0

package/README.md

@@ -46,7 +46,26 @@ yarn add embedoc
 
 ## Quick Start
 
-### 1. Create Configuration File
+### 1. Initialize Project
+
+```bash
+npx embedoc init
+```
+
+This creates:
+- `embedoc.config.yaml` - configuration file
+- `.embedoc/renderers/index.ts` - renderer registration
+- `.embedoc/datasources/index.ts` - custom datasource registration
+- `.embedoc/templates/` - Handlebars templates directory
+
+If `package.json` exists, npm scripts are also added:
+- `npm run embedoc:build` - build documents
+- `npm run embedoc:watch` - watch mode
+- `npm run embedoc:generate` - run generators
+
+### 2. Configure
+
+Edit `embedoc.config.yaml` to set your targets and datasources:
 
 ```yaml
 # embedoc.config.yaml
@@ -62,15 +81,12 @@ datasources:
   metadata_db:
     type: sqlite
     path: "./data/metadata.db"
-
-embeds_dir: "./embeds"
-templates_dir: "./templates"
 ```
 
-### 2. Create an Embed
+### 3. Create a Renderer
 
 ```typescript
-// embeds/table_columns.ts
+// .embedoc/renderers/table_columns.ts
 import { defineEmbed } from 'embedoc';
 
 export default defineEmbed({
@@ -104,10 +120,10 @@ export default defineEmbed({
 });
 ```
 
-Register your embed in `embeds/index.ts`:
+Register your renderer in `.embedoc/renderers/index.ts`:
 
 ```typescript
-// embeds/index.ts
+// .embedoc/renderers/index.ts
 import tableColumns from './table_columns.ts';
 
 export const embeds = {
@@ -117,7 +133,7 @@ export const embeds = {
 
 > **Note**: embedoc can directly import TypeScript files, so **no compilation is required**.
 
-### 3. Add Markers to Your Document
+### 4. Add Markers to Your Document
 
 ```markdown
 # Users Table
@@ -126,10 +142,12 @@ export const embeds = {
 <!--@embedoc:end-->
 ```
 
-### 4. Run Build
+### 5. Run Build
 
 ```bash
 npx embedoc build
+# or, if scripts were added to package.json:
+npm run embedoc:build
 ```
 
 ---
@@ -137,7 +155,12 @@ npx embedoc build
 ## CLI Commands
 
 ```bash
+# Initialize project (creates config, .embedoc/ directory, updates package.json)
+embedoc init
+embedoc init --force   # overwrite existing files
+
 # Build all files
+embedoc build
 embedoc build --config embedoc.config.yaml
 
 # Build specific files only
@@ -150,6 +173,7 @@ embedoc generate --datasource tables
 embedoc generate --all
 
 # Watch mode (incremental build)
+embedoc watch
 embedoc watch --config embedoc.config.yaml
 
 # Debug dependency graph
@@ -162,6 +186,8 @@ embedoc build --dry-run
 embedoc build --verbose
 ```
 
+All commands can be run directly with `npx embedoc <command>` or via package.json scripts after `embedoc init`.
+
 ---
 
 ## Configuration File
@@ -246,11 +272,14 @@ datasources:
     type: glob
     pattern: "./docs/**/*.md"
 
-# Embed directory (TypeScript)
-embeds_dir: "./embeds"
+# Renderer directory (TypeScript) - default: ".embedoc/renderers"
+renderers_dir: ".embedoc/renderers"
+
+# Custom datasource types directory - default: ".embedoc/datasources"
+datasources_dir: ".embedoc/datasources"
 
-# Template directory (Handlebars)
-templates_dir: "./templates"
+# Template directory (Handlebars) - default: ".embedoc/templates"
+templates_dir: ".embedoc/templates"
 
 # Output settings
 output:
@@ -713,6 +742,97 @@ inline_datasource:
 
 ---
 
+## Custom Datasources
+
+Define custom datasource types in TypeScript to connect to any data source (APIs, databases, custom file formats, etc.).
+
+### Defining a Custom Datasource Type
+
+```typescript
+// .embedoc/datasources/github.ts
+import { defineDatasource } from 'embedoc';
+
+export default defineDatasource({
+  async create(config) {
+    const owner = config['owner'] as string;
+    const repo = config['repo'] as string;
+    const response = await fetch(
+      `https://api.github.com/repos/${owner}/${repo}/issues`
+    );
+    const issues = await response.json();
+
+    return {
+      async query() { return issues; },
+      async getAll() { return issues; },
+      async close() {},
+    };
+  }
+});
+```
+
+### Registering Custom Datasource Types
+
+```typescript
+// .embedoc/datasources/index.ts
+import github from './github.ts';
+
+export const datasourceTypes = {
+  github,
+};
+```
+
+### Using in Configuration
+
+```yaml
+# embedoc.config.yaml
+datasources:
+  my_issues:
+    type: github         # matches key in datasourceTypes
+    owner: "myorg"
+    repo: "myrepo"
+```
+
+All config properties are passed to the `create()` method, so custom datasources can accept any configuration.
+
+### Custom Inline Format Parsers
+
+Register custom format parsers for `@embedoc-data` inline markers by exporting `inlineFormats` alongside `datasourceTypes`:
+
+```typescript
+// .embedoc/datasources/index.ts
+import github from './github.ts';
+
+export const datasourceTypes = {
+  github,
+};
+
+export const inlineFormats = {
+  toml: (content: string) => parseToml(content),
+  ini: (content: string) => {
+    const result: Record<string, string> = {};
+    for (const line of content.split('\n')) {
+      const [key, ...rest] = line.split('=');
+      if (key && rest.length > 0) {
+        result[key.trim()] = rest.join('=').trim();
+      }
+    }
+    return result;
+  },
+};
+```
+
+Custom formats can then be used in documents:
+
+```markdown
+<!--@embedoc-data:config format="ini"-->
+host=localhost
+port=5432
+database=myapp
+<!--@embedoc-data:end-->
+```
+
+---
+
 ## File Generation
 
 Generate new files in bulk using Handlebars templates based on datasource records.
@@ -734,7 +854,7 @@ datasources:
 ### Template (Handlebars)
 
 ```handlebars
-{{!-- templates/table_doc.hbs --}}
+{{!-- .embedoc/templates/table_doc.hbs --}}
 ---
 doc_id: "{{table_name}}"
 embeds:
@@ -851,26 +971,31 @@ Frontmatter values can be referenced in marker attributes using `${...}` syntax.
 
 ```
 your-project/
-├── embedoc.config.yaml     # Configuration file
-├── embeds/                  # Embed definitions (TypeScript)
-│   ├── table_columns.ts
-│   ├── table_relations.ts
-│   └── index.ts             # Export all embeds
-├── templates/               # File generation templates (Handlebars)
-│   ├── table_doc.hbs
-│   └── view_doc.hbs
-├── data/                    # Datasources
+├── embedoc.config.yaml        # Configuration file
+├── .embedoc/                   # All embedoc custom code
+│   ├── renderers/              # Renderer definitions (TypeScript)
+│   │   ├── index.ts            # Export all renderers
+│   │   ├── table_columns.ts
+│   │   └── table_relations.ts
+│   ├── datasources/            # Custom datasource types (TypeScript)
+│   │   ├── index.ts            # Export datasourceTypes and inlineFormats
+│   │   └── github.ts
+│   ├── templates/              # File generation templates (Handlebars)
+│   │   ├── table_doc.hbs
+│   │   └── view_doc.hbs
+│   └── package.json            # Optional: dependencies for custom code
+├── data/                       # Datasource files
 │   ├── metadata.db
 │   └── endpoints.csv
-└── docs/                    # Target documents
+└── docs/                       # Target documents
     └── tables/
         └── users.md
 ```
 
-### Embed Registration
+### Renderer Registration
 
 ```typescript
-// embeds/index.ts
+// .embedoc/renderers/index.ts
 import tableColumns from './table_columns.ts';
 import tableRelations from './table_relations.ts';
 import customEmbed from './custom_embed.ts';
@@ -923,6 +1048,7 @@ npm test
 import {
   // Core
   defineEmbed,
+  defineDatasource,
   build,
   processFile,
   
@@ -1002,6 +1128,12 @@ interface InlineDatasourceConfig {
   stripCodeFences?: boolean;
   stripPatterns?: string[];
 }
+
+interface CustomDatasourceDefinition {
+  create(config: DatasourceConfig): Promise<Datasource>;
+}
+
+type InlineFormatParser = (content: string) => unknown;
 ```
 
 ---

package/dist/cli.js

@@ -3,7 +3,7 @@
  * embedoc CLI
  */
 import { Command } from 'commander';
-import { readFile, access } from 'node:fs/promises';
+import { readFile, writeFile, access, mkdir } from 'node:fs/promises';
 import { resolve, relative } from 'node:path';
 import { pathToFileURL } from 'node:url';
 import yaml from 'js-yaml';
@@ -18,7 +18,7 @@ const program = new Command();
 program
     .name('embedoc')
     .description('In-Place Document Generator')
-    .version('0.1.0');
+    .version('0.11.0');
 /**
  * Load configuration file
  */
@@ -43,34 +43,208 @@ async function fileExists(filePath) {
     }
 }
 /**
- * Load embeds (supports both TypeScript and JavaScript)
+ * Resolve renderers_dir from config, with embeds_dir as deprecated fallback
  */
-async function loadEmbeds(embedsDir) {
-    const tsIndexPath = resolve(embedsDir, 'index.ts');
-    const jsIndexPath = resolve(embedsDir, 'index.js');
+function resolveRenderersDir(config) {
+    if (config.renderers_dir) {
+        return config.renderers_dir;
+    }
+    if (config.embeds_dir) {
+        console.warn(pc.yellow('Warning: "embeds_dir" is deprecated. Use "renderers_dir" instead.'));
+        return config.embeds_dir;
+    }
+    return '.embedoc/renderers';
+}
+/**
+ * Load renderers (supports both TypeScript and JavaScript)
+ */
+async function loadRenderers(renderersDir) {
+    const tsIndexPath = resolve(renderersDir, 'index.ts');
+    const jsIndexPath = resolve(renderersDir, 'index.js');
     try {
-        // First try TypeScript file
         if (await fileExists(tsIndexPath)) {
-            // Use tsx to directly import TypeScript
-            // tsImport may return { default: { embeds: ... } } or { embeds: ... }
             const module = await tsImport(tsIndexPath, import.meta.url);
             return module.embeds ?? module.default?.embeds ?? {};
         }
-        // Fall back to JavaScript
         if (await fileExists(jsIndexPath)) {
             const moduleUrl = pathToFileURL(jsIndexPath).href;
             const module = (await import(moduleUrl));
             return module.embeds ?? {};
         }
-        console.warn(pc.yellow(`Warning: No embeds found in ${embedsDir} (index.ts or index.js)`));
         return {};
     }
     catch (error) {
         const message = error instanceof Error ? error.message : String(error);
-        console.warn(pc.yellow(`Warning: Could not load embeds from ${embedsDir}: ${message}`));
+        console.warn(pc.yellow(`Warning: Could not load renderers from ${renderersDir}: ${message}`));
         return {};
     }
 }
+/**
+ * Load custom datasource types and inline format parsers from datasources_dir
+ */
+async function loadCustomDatasources(datasourcesDir) {
+    const tsIndexPath = resolve(datasourcesDir, 'index.ts');
+    const jsIndexPath = resolve(datasourcesDir, 'index.js');
+    const empty = { datasourceTypes: {}, inlineFormats: {} };
+    try {
+        let module;
+        if (await fileExists(tsIndexPath)) {
+            module = await tsImport(tsIndexPath, import.meta.url);
+        }
+        else if (await fileExists(jsIndexPath)) {
+            const moduleUrl = pathToFileURL(jsIndexPath).href;
+            module = (await import(moduleUrl));
+        }
+        if (!module) {
+            return empty;
+        }
+        return {
+            datasourceTypes: module.datasourceTypes ?? module.default?.datasourceTypes ?? {},
+            inlineFormats: module.inlineFormats ?? module.default?.inlineFormats ?? {},
+        };
+    }
+    catch (error) {
+        const message = error instanceof Error ? error.message : String(error);
+        console.warn(pc.yellow(`Warning: Could not load custom datasources from ${datasourcesDir}: ${message}`));
+        return empty;
+    }
+}
+/**
+ * init command
+ */
+program
+    .command('init')
+    .description('Initialize embedoc in the current project')
+    .option('-f, --force', 'Overwrite existing files')
+    .action(async (options) => {
+    try {
+        console.log(pc.cyan('📁 Initializing embedoc...\n'));
+        const created = [];
+        const skipped = [];
+        const writeIfNotExists = async (filePath, content) => {
+            const absPath = resolve(filePath);
+            if (!options.force && await fileExists(absPath)) {
+                skipped.push(filePath);
+                return;
+            }
+            const dir = absPath.substring(0, absPath.lastIndexOf('/'));
+            await mkdir(dir, { recursive: true });
+            await writeFile(absPath, content, { encoding: 'utf-8' });
+            created.push(filePath);
+        };
+        // embedoc.config.yaml
+        const configContent = `version: "1.0"
+
+targets:
+  - pattern: "./docs/**/*.md"
+    comment_style: html
+    exclude:
+      - "**/node_modules/**"
+      - "**/.git/**"
+
+datasources: {}
+  # example_db:
+  #   type: sqlite
+  #   path: "./data/example.db"
+  # example_csv:
+  #   type: csv
+  #   path: "./data/example.csv"
+`;
+        await writeIfNotExists('embedoc.config.yaml', configContent);
+        // .embedoc/renderers/index.ts
+        const renderersIndexContent = `/**
+ * embedoc renderers
+ *
+ * Export your custom renderers here.
+ * Each key becomes a marker name: <!--@embedoc:key_name ...-->
+ */
+
+// import myRenderer from './my_renderer.ts';
+
+export const embeds = {
+  // my_renderer: myRenderer,
+};
+`;
+        await writeIfNotExists('.embedoc/renderers/index.ts', renderersIndexContent);
+        // .embedoc/datasources/index.ts
+        const datasourcesIndexContent = `/**
+ * embedoc custom datasources and inline format parsers
+ *
+ * datasourceTypes: register custom datasource types usable in embedoc.config.yaml
+ * inlineFormats: register custom parsers for @embedoc-data format="xxx" markers
+ */
+
+// import type { CustomDatasourceDefinition, InlineFormatParser } from 'embedoc';
+// import myDatasource from './my_datasource.ts';
+
+export const datasourceTypes = {
+  // my_datasource: myDatasource,
+};
+
+export const inlineFormats = {
+  // toml: (content: string) => parseToml(content),
+};
+`;
+        await writeIfNotExists('.embedoc/datasources/index.ts', datasourcesIndexContent);
+        // .embedoc/templates/ (create directory only via a .gitkeep)
+        await writeIfNotExists('.embedoc/templates/.gitkeep', '');
+        // Update package.json if it exists
+        const pkgPath = resolve('package.json');
+        if (await fileExists(pkgPath)) {
+            const pkgRaw = await readFile(pkgPath, { encoding: 'utf-8' });
+            const pkg = JSON.parse(pkgRaw);
+            const scripts = (pkg['scripts'] ?? {});
+            let scriptsUpdated = false;
+            const scriptEntries = [
+                ['embedoc:build', 'embedoc build'],
+                ['embedoc:watch', 'embedoc watch'],
+                ['embedoc:generate', 'embedoc generate --all'],
+            ];
+            for (const [key, value] of scriptEntries) {
+                if (!(key in scripts)) {
+                    scripts[key] = value;
+                    scriptsUpdated = true;
+                }
+            }
+            if (scriptsUpdated) {
+                pkg['scripts'] = scripts;
+                await writeFile(pkgPath, JSON.stringify(pkg, null, 2) + '\n', { encoding: 'utf-8' });
+                console.log(pc.green('   Updated package.json scripts:'));
+                for (const [key, value] of scriptEntries) {
+                    if (!((pkgRaw.includes(`"${key}"`)))) {
+                        console.log(pc.gray(`     "${key}": "${value}"`));
+                    }
+                }
+                console.log('');
+            }
+        }
+        // Summary
+        if (created.length > 0) {
+            console.log(pc.green('   Created:'));
+            for (const f of created) {
+                console.log(pc.gray(`     ${f}`));
+            }
+        }
+        if (skipped.length > 0) {
+            console.log(pc.yellow('   Skipped (already exists):'));
+            for (const f of skipped) {
+                console.log(pc.gray(`     ${f}`));
+            }
+        }
+        console.log('');
+        console.log(pc.green('✅ embedoc initialized!'));
+        console.log('');
+        console.log('   Next steps:');
+        console.log(pc.cyan('   1. Edit embedoc.config.yaml to configure targets and datasources'));
+        console.log(pc.cyan('   2. Add renderers in .embedoc/renderers/'));
+        console.log(pc.cyan('   3. Run: npx embedoc build'));
+    }
+    catch (error) {
+        console.error(pc.red('❌ Init failed:'));
+        console.error(error instanceof Error ? error.message : String(error));
+        process.exit(1);
+    }
+});
 /**
  * build command
  */
@@ -85,16 +259,19 @@ program
     try {
         console.log(pc.cyan('🔧 Loading configuration...'));
         const config = await loadConfig(options.config);
+        const datasourcesDir = config.datasources_dir ?? '.embedoc/datasources';
+        const customModules = await loadCustomDatasources(datasourcesDir);
         console.log(pc.cyan('📦 Initializing datasources...'));
-        const datasources = initializeDatasources(config);
-        console.log(pc.cyan('📝 Loading embeds...'));
-        const embedsDir = config.embeds_dir ?? './embeds';
-        const embeds = await loadEmbeds(embedsDir);
+        const datasources = await initializeDatasources(config, customModules.datasourceTypes);
+        console.log(pc.cyan('📝 Loading renderers...'));
+        const renderersDir = resolveRenderersDir(config);
+        const embeds = await loadRenderers(renderersDir);
         console.log(pc.cyan('🔄 Processing files...'));
         const result = await build(config, embeds, datasources, {
             dryRun: options.dryRun,
             verbose: options.verbose,
             specificFiles: files.length > 0 ? files : undefined,
+            customInlineFormats: customModules.inlineFormats,
         });
         // Cleanup
         await closeDatasources(datasources);
@@ -148,8 +325,10 @@ program
         }
         console.log(pc.cyan('🔧 Loading configuration...'));
         const config = await loadConfig(options.config);
+        const datasourcesDir = config.datasources_dir ?? '.embedoc/datasources';
+        const customModules = await loadCustomDatasources(datasourcesDir);
         console.log(pc.cyan('📦 Initializing datasources...'));
-        const datasources = initializeDatasources(config);
+        const datasources = await initializeDatasources(config, customModules.datasourceTypes);
         console.log(pc.cyan('📄 Generating files...'));
         const results = await generateAll(config, datasources, {
             dryRun: options.dryRun,
@@ -203,11 +382,13 @@ program
     try {
         console.log(pc.cyan('🔧 Loading configuration...'));
         const config = await loadConfig(options.config);
+        const datasourcesDir = config.datasources_dir ?? '.embedoc/datasources';
+        const customModules = await loadCustomDatasources(datasourcesDir);
         console.log(pc.cyan('📦 Initializing datasources...'));
-        let datasources = initializeDatasources(config);
-        console.log(pc.cyan('📝 Loading embeds...'));
-        const embedsDir = resolve(config.embeds_dir ?? './embeds');
-        let embeds = await loadEmbeds(embedsDir);
+        let datasources = await initializeDatasources(config, customModules.datasourceTypes);
+        console.log(pc.cyan('📝 Loading renderers...'));
+        const renderersDir = resolve(resolveRenderersDir(config));
+        let embeds = await loadRenderers(renderersDir);
         // Build dependency graph
         console.log(pc.cyan('🔗 Building dependency graph...'));
         let depGraph = new DependencyGraph(config, embeds);
@@ -234,25 +415,23 @@ program
         const processChanges = async () => {
             const changes = new Map(pendingChanges);
             pendingChanges.clear();
-            // Reload embeds if embed file changed
-            let embedsReloaded = false;
+            let renderersReloaded = false;
             for (const [filePath] of changes) {
-                if (filePath.startsWith(embedsDir) && filePath.endsWith('.ts')) {
-                    console.log(pc.yellow(`\n🔄 Embed changed: ${relative(process.cwd(), filePath)}`));
-                    console.log(pc.cyan('   Reloading embeds...'));
+                if (filePath.startsWith(renderersDir) && filePath.endsWith('.ts')) {
+                    console.log(pc.yellow(`\n🔄 Renderer changed: ${relative(process.cwd(), filePath)}`));
+                    console.log(pc.cyan('   Reloading renderers...'));
                     try {
-                        // Import with new timestamp to clear tsx cache
-                        embeds = await loadEmbeds(embedsDir);
-                        embedsReloaded = true;
+                        embeds = await loadRenderers(renderersDir);
+                        renderersReloaded = true;
                     }
                     catch (error) {
-                        console.error(pc.red('   Failed to reload embeds:'), error);
+                        console.error(pc.red('   Failed to reload renderers:'), error);
                         return;
                     }
                 }
             }
             // Rebuild dependency graph if embeds were reloaded
-            if (embedsReloaded) {
+            if (renderersReloaded) {
                 console.log(pc.cyan('   Rebuilding dependency graph...'));
                 depGraph = new DependencyGraph(config, embeds);
                 await depGraph.build();
@@ -291,10 +470,11 @@ program
             try {
                 // Re-initialize datasources (especially for SQLite)
                 await closeDatasources(datasources);
-                datasources = initializeDatasources(config);
+                datasources = await initializeDatasources(config, customModules.datasourceTypes);
                 const result = await build(config, embeds, datasources, {
                     verbose: options.verbose,
                     specificFiles: Array.from(affectedDocs),
+                    customInlineFormats: customModules.inlineFormats,
                 });
                 if (result.totalMarkersUpdated > 0) {
                     console.log(pc.green(`   ✅ Updated ${result.totalMarkersUpdated} marker(s)`));