embedoc 0.10.2 → 0.11.1

package/README.md

@@ -46,7 +46,26 @@ yarn add embedoc
 
 ## Quick Start
 
-### 1. Create Configuration File
+### 1. Initialize Project
+
+```bash
+npx embedoc init
+```
+
+This creates:
+- `embedoc.config.yaml` - configuration file
+- `.embedoc/renderers/index.ts` - renderer registration
+- `.embedoc/datasources/index.ts` - custom datasource registration
+- `.embedoc/templates/` - Handlebars templates directory
+
+If `package.json` exists, npm scripts are also added:
+- `npm run embedoc:build` - build documents
+- `npm run embedoc:watch` - watch mode
+- `npm run embedoc:generate` - run generators
+
+### 2. Configure
+
+Edit `embedoc.config.yaml` to set your targets and datasources:
 
 ```yaml
 # embedoc.config.yaml
@@ -62,15 +81,12 @@ datasources:
   metadata_db:
     type: sqlite
     path: "./data/metadata.db"
-
-embeds_dir: "./embeds"
-templates_dir: "./templates"
 ```
 
-### 2. Create an Embed
+### 3. Create a Renderer
 
 ```typescript
-// embeds/table_columns.ts
+// .embedoc/renderers/table_columns.ts
 import { defineEmbed } from 'embedoc';
 
 export default defineEmbed({
@@ -104,10 +120,10 @@ export default defineEmbed({
 });
 ```
 
-Register your embed in `embeds/index.ts`:
+Register your renderer in `.embedoc/renderers/index.ts`:
 
 ```typescript
-// embeds/index.ts
+// .embedoc/renderers/index.ts
 import tableColumns from './table_columns.ts';
 
 export const embeds = {
@@ -117,7 +133,7 @@ export const embeds = {
 
 > **Note**: embedoc can directly import TypeScript files, so **no compilation is required**.
 
-### 3. Add Markers to Your Document
+### 4. Add Markers to Your Document
 
 ```markdown
 # Users Table
@@ -126,18 +142,27 @@ export const embeds = {
 <!--@embedoc:end-->
 ```
 
-### 4. Run Build
+### 5. Run Build
 
 ```bash
 npx embedoc build
+# or, if scripts were added to package.json:
+npm run embedoc:build
 ```
 
 ---
 
 ## CLI Commands
 
+> **Full CLI reference:** [docs/cli-reference.md](./docs/cli-reference.md) | **Machine-readable contract:** [cli-contract.yaml](./cli-contract.yaml)
+
 ```bash
+# Initialize project (creates config, .embedoc/ directory, updates package.json)
+embedoc init
+embedoc init --force   # overwrite existing files
+
 # Build all files
+embedoc build
 embedoc build --config embedoc.config.yaml
 
 # Build specific files only
@@ -146,10 +171,14 @@ embedoc build ./path/to/file.md
 # Generate new files (specific datasource)
 embedoc generate --datasource tables
 
+# Generate with specific template
+embedoc generate --datasource tables --generator table_doc.hbs
+
 # Run all datasource generators
 embedoc generate --all
 
 # Watch mode (incremental build)
+embedoc watch
 embedoc watch --config embedoc.config.yaml
 
 # Debug dependency graph
@@ -162,6 +191,8 @@ embedoc build --dry-run
 embedoc build --verbose
 ```
 
+All commands can be run directly with `npx embedoc <command>` or via package.json scripts after `embedoc init`.
+
 ---
 
 ## Configuration File
@@ -246,11 +277,14 @@ datasources:
     type: glob
     pattern: "./docs/**/*.md"
 
-# Embed directory (TypeScript)
-embeds_dir: "./embeds"
+# Renderer directory (TypeScript) - default: ".embedoc/renderers"
+renderers_dir: ".embedoc/renderers"
 
-# Template directory (Handlebars)
-templates_dir: "./templates"
+# Custom datasource types directory - default: ".embedoc/datasources"
+datasources_dir: ".embedoc/datasources"
+
+# Template directory (Handlebars) - default: ".embedoc/templates"
+templates_dir: ".embedoc/templates"
 
 # Output settings
 output:
@@ -713,6 +747,97 @@ inline_datasource:
 
 ---
 
+## Custom Datasources
+
+Define custom datasource types in TypeScript to connect to any data source (APIs, databases, custom file formats, etc.).
+
+### Defining a Custom Datasource Type
+
+```typescript
+// .embedoc/datasources/github.ts
+import { defineDatasource } from 'embedoc';
+
+export default defineDatasource({
+  async create(config) {
+    const owner = config['owner'] as string;
+    const repo = config['repo'] as string;
+    const response = await fetch(
+      `https://api.github.com/repos/${owner}/${repo}/issues`
+    );
+    const issues = await response.json();
+
+    return {
+      async query() { return issues; },
+      async getAll() { return issues; },
+      async close() {},
+    };
+  }
+});
+```
+
+### Registering Custom Datasource Types
+
+```typescript
+// .embedoc/datasources/index.ts
+import github from './github.ts';
+
+export const datasourceTypes = {
+  github,
+};
+```
+
+### Using in Configuration
+
+```yaml
+# embedoc.config.yaml
+datasources:
+  my_issues:
+    type: github         # matches key in datasourceTypes
+    owner: "myorg"
+    repo: "myrepo"
+```
+
+All config properties are passed to the `create()` method, so custom datasources can accept any configuration.
+
+### Custom Inline Format Parsers
+
+Register custom format parsers for `@embedoc-data` inline markers by exporting `inlineFormats` alongside `datasourceTypes`:
+
+```typescript
+// .embedoc/datasources/index.ts
+import github from './github.ts';
+
+export const datasourceTypes = {
+  github,
+};
+
+export const inlineFormats = {
+  toml: (content: string) => parseToml(content),
+  ini: (content: string) => {
+    const result: Record<string, string> = {};
+    for (const line of content.split('\n')) {
+      const [key, ...rest] = line.split('=');
+      if (key && rest.length > 0) {
+        result[key.trim()] = rest.join('=').trim();
+      }
+    }
+    return result;
+  },
+};
+```
+
+Custom formats can then be used in documents:
+
+```markdown
+<!--@embedoc-data:config format="ini"-->
+host=localhost
+port=5432
+database=myapp
+<!--@embedoc-data:end-->
+```
+
+---
+
 ## File Generation
 
 Generate new files in bulk using Handlebars templates based on datasource records.
@@ -734,7 +859,7 @@ datasources:
 ### Template (Handlebars)
 
 ```handlebars
-{{!-- templates/table_doc.hbs --}}
+{{!-- .embedoc/templates/table_doc.hbs --}}
 ---
 doc_id: "{{table_name}}"
 embeds:
@@ -851,26 +976,31 @@ Frontmatter values can be referenced in marker attributes using `${...}` syntax.
 
 ```
 your-project/
-├── embedoc.config.yaml     # Configuration file
-├── embeds/                  # Embed definitions (TypeScript)
-│   ├── table_columns.ts
-│   ├── table_relations.ts
-│   └── index.ts             # Export all embeds
-├── templates/               # File generation templates (Handlebars)
-│   ├── table_doc.hbs
-│   └── view_doc.hbs
-├── data/                    # Datasources
+├── embedoc.config.yaml        # Configuration file
+├── .embedoc/                   # All embedoc custom code
+│   ├── renderers/              # Renderer definitions (TypeScript)
+│   │   ├── index.ts            # Export all renderers
+│   │   ├── table_columns.ts
+│   │   └── table_relations.ts
+│   ├── datasources/            # Custom datasource types (TypeScript)
+│   │   ├── index.ts            # Export datasourceTypes and inlineFormats
+│   │   └── github.ts
+│   ├── templates/              # File generation templates (Handlebars)
+│   │   ├── table_doc.hbs
+│   │   └── view_doc.hbs
+│   └── package.json            # Optional: dependencies for custom code
+├── data/                       # Datasource files
 │   ├── metadata.db
 │   └── endpoints.csv
-└── docs/                    # Target documents
+└── docs/                       # Target documents
     └── tables/
         └── users.md
 ```
 
-### Embed Registration
+### Renderer Registration
 
 ```typescript
-// embeds/index.ts
+// .embedoc/renderers/index.ts
 import tableColumns from './table_columns.ts';
 import tableRelations from './table_relations.ts';
 import customEmbed from './custom_embed.ts';
@@ -923,6 +1053,7 @@ npm test
 import {
   // Core
   defineEmbed,
+  defineDatasource,
   build,
   processFile,
   
@@ -1002,6 +1133,12 @@ interface InlineDatasourceConfig {
   stripCodeFences?: boolean;
   stripPatterns?: string[];
 }
+
+interface CustomDatasourceDefinition {
+  create(config: DatasourceConfig): Promise<Datasource>;
+}
+
+type InlineFormatParser = (content: string) => unknown;
 ```
 
 ---

package/cli-contract.yaml

@@ -0,0 +1,327 @@
+# yaml-language-server: $schema=./node_modules/cli-contracts/schemas/cli-contract.schema.json
+cliContracts: 0.1.0
+
+info:
+  title: embedoc CLI
+  version: 0.11.1
+  description: >-
+    In-place document generator that auto-updates marker blocks in documents
+    and source code while preserving manually edited sections. Supports
+    multiple datasources (SQLite, CSV, JSON, YAML, glob), programmable
+    TypeScript renderers, Handlebars file generation, and incremental builds
+    with dependency tracking.
+  license:
+    name: MIT
+  contact:
+    name: foo-ogawa
+    url: https://github.com/foo-ogawa/embedoc
+
+commandSets:
+  embedoc:
+    summary: In-place document generator with marker-based embedding.
+    executable: embedoc
+
+    globalOptions:
+      - name: version
+        aliases: [V]
+        description: Print version and exit.
+        schema:
+          type: boolean
+
+      - name: help
+        aliases: [h]
+        description: Show help and exit.
+        schema:
+          type: boolean
+
+    commands:
+      # ── init ──────────────────────────────────────────
+      init:
+        summary: Initialize embedoc in the current project.
+        description: >-
+          Creates starter configuration and directory structure:
+          embedoc.config.yaml, .embedoc/renderers/index.ts,
+          .embedoc/datasources/index.ts, and .embedoc/templates/.
+          If package.json exists, adds embedoc:build, embedoc:watch,
+          and embedoc:generate npm scripts.
+        usage:
+          - embedoc init
+          - embedoc init --force
+
+        options:
+          - name: force
+            aliases: [f]
+            description: Overwrite existing files.
+            schema:
+              type: boolean
+              default: false
+
+        exits:
+          '0':
+            description: Initialization succeeded.
+            stdout:
+              format: text
+            files:
+              - path: embedoc.config.yaml
+                required: false
+                mediaType: application/yaml
+                encoding: utf-8
+                description: Configuration file.
+              - path: .embedoc/renderers/index.ts
+                required: false
+                mediaType: text/x-typescript
+                encoding: utf-8
+                description: Renderer registration file.
+              - path: .embedoc/datasources/index.ts
+                required: false
+                mediaType: text/x-typescript
+                encoding: utf-8
+                description: Custom datasource registration file.
+              - path: .embedoc/templates/.gitkeep
+                required: false
+                description: Templates directory placeholder.
+
+          '1':
+            description: Initialization failed (write error).
+            stderr:
+              format: text
+
+        x-agent:
+          riskLevel: low
+          requiresConfirmation: false
+          idempotent: true
+          sideEffects:
+            - file_write
+            - directory_create
+
+      # ── build ─────────────────────────────────────────
+      build:
+        summary: Build documents by replacing markers with rendered content.
+        description: >-
+          Loads configuration, initializes datasources, loads TypeScript
+          renderers, and processes all target files (or specific files if
+          provided). Each @embedoc marker block is re-rendered and the
+          content between start/end markers is replaced in-place.
+        usage:
+          - embedoc build
+          - embedoc build --config embedoc.config.yaml
+          - embedoc build ./docs/tables/users.md
+          - embedoc build --dry-run --verbose
+
+        arguments:
+          - name: files
+            description: Specific file paths to process. If omitted, processes all targets from config.
+            schema:
+              type: array
+              items:
+                type: string
+            variadic: true
+            file:
+              mode: readWrite
+              exists: true
+
+        options:
+          - name: config
+            aliases: [c]
+            description: Path to config file (YAML or JSON).
+            valueName: path
+            schema:
+              type: string
+              default: embedoc.config.yaml
+            file:
+              mode: read
+              exists: true
+              mediaType: application/yaml
+              encoding: utf-8
+
+          - name: dry-run
+            aliases: [d]
+            description: Preview changes without writing files.
+            schema:
+              type: boolean
+              default: false
+
+          - name: verbose
+            aliases: [v]
+            description: Enable verbose output.
+            schema:
+              type: boolean
+              default: false
+
+        exits:
+          '0':
+            description: Build succeeded. All markers were processed without errors.
+            stdout:
+              format: text
+
+          '1':
+            description: Build failed (config not found, datasource error, or marker processing error).
+            stderr:
+              format: text
+
+        x-agent:
+          riskLevel: medium
+          requiresConfirmation: false
+          idempotent: true
+          sideEffects:
+            - file_write
+          safeDryRunOption: --dry-run
+          recommendedBeforeUse:
+            - Ensure embedoc.config.yaml exists (run init if needed).
+            - Verify datasource files are accessible.
+
+      # ── generate ──────────────────────────────────────
+      generate:
+        summary: Generate new files from datasource records using Handlebars templates.
+        description: >-
+          Reads datasource records and generates new files using Handlebars
+          templates defined in the generators config. Requires either
+          --datasource to process a specific datasource or --all to process
+          all datasources that have generators configured. Optionally filter
+          by generator template name with --generator.
+        usage:
+          - embedoc generate --datasource tables
+          - embedoc generate --all
+          - embedoc generate --datasource tables --generator table_doc.hbs
+          - embedoc generate --all --dry-run --verbose
+
+        options:
+          - name: config
+            aliases: [c]
+            description: Path to config file (YAML or JSON).
+            valueName: path
+            schema:
+              type: string
+              default: embedoc.config.yaml
+            file:
+              mode: read
+              exists: true
+              mediaType: application/yaml
+              encoding: utf-8
+
+          - name: datasource
+            aliases: [s]
+            description: Specific datasource to process.
+            valueName: name
+            schema:
+              type: string
+
+          - name: generator
+            aliases: [g]
+            description: Specific generator template to use.
+            valueName: name
+            schema:
+              type: string
+
+          - name: all
+            aliases: [a]
+            description: Process all datasources that have generators configured.
+            schema:
+              type: boolean
+              default: false
+
+          - name: dry-run
+            aliases: [d]
+            description: Preview generation without writing files.
+            schema:
+              type: boolean
+              default: false
+
+          - name: verbose
+            aliases: [v]
+            description: Enable verbose output.
+            schema:
+              type: boolean
+              default: false
+
+        exits:
+          '0':
+            description: Generation succeeded.
+            stdout:
+              format: text
+
+          '1':
+            description: >-
+              Generation failed (missing --datasource/--all, config not found,
+              datasource error, or template error).
+            stderr:
+              format: text
+
+        x-agent:
+          riskLevel: medium
+          requiresConfirmation: false
+          idempotent: true
+          sideEffects:
+            - file_write
+          safeDryRunOption: --dry-run
+          recommendedBeforeUse:
+            - Ensure embedoc.config.yaml exists with generators configured.
+            - Verify datasource files are accessible.
+            - Ensure Handlebars templates exist in templates_dir.
+
+      # ── watch ─────────────────────────────────────────
+      watch:
+        summary: Watch files and rebuild on changes with dependency tracking.
+        description: >-
+          Monitors target files, renderers, and datasource files for changes.
+          When a change is detected, rebuilds only affected documents using
+          the dependency graph. Renderer changes trigger a full renderer
+          reload and dependency graph rebuild. Runs until interrupted with
+          SIGINT (Ctrl+C).
+        usage:
+          - embedoc watch
+          - embedoc watch --config embedoc.config.yaml
+          - embedoc watch --verbose
+          - embedoc watch --debug-deps
+
+        options:
+          - name: config
+            aliases: [c]
+            description: Path to config file (YAML or JSON).
+            valueName: path
+            schema:
+              type: string
+              default: embedoc.config.yaml
+            file:
+              mode: read
+              exists: true
+              mediaType: application/yaml
+              encoding: utf-8
+
+          - name: verbose
+            aliases: [v]
+            description: Enable verbose rebuild output.
+            schema:
+              type: boolean
+              default: false
+
+          - name: debug-deps
+            description: Print dependency graph at startup for debugging.
+            schema:
+              type: boolean
+              default: false
+
+        signals:
+          SIGINT:
+            description: Gracefully stops the watcher, closes datasources, and exits.
+
+        exits:
+          '0':
+            description: Watcher stopped gracefully (via SIGINT).
+            stdout:
+              format: text
+
+          '1':
+            description: Watch failed (config not found or initialization error).
+            stderr:
+              format: text
+
+        x-agent:
+          riskLevel: medium
+          requiresConfirmation: false
+          idempotent: true
+          sideEffects:
+            - file_write
+          recommendedBeforeUse:
+            - Ensure embedoc.config.yaml exists (run init if needed).
+            - Verify datasource files are accessible.