@c7-digital/scribe 0.0.1

package/README.md

@@ -0,0 +1,336 @@
+# @c7-digital/scribe
+
+Transform raw DAML codegen output into unified, modern ESM TypeScript packages for use with `@c7-digital/ledger`.
+
+## What it does
+
+`dpm codegen-js` produces raw TypeScript/CommonJS packages -- one per DAR -- with global template registration, mixed import conventions, and self-referencing type imports. Scribe absorbs all of that complexity:
+
+- **CJS to ESM**: bundles CommonJS `require()`/`module.exports` into clean ESM `import`/`export` via Rollup
+- **Global registry bypass**: strips `damlTypes.registerTemplate()` side-effects, replaces them with a version-aware registry
+- **Import fixups**: rewrites `@mojotech/json-type-validation` (default -> namespace), `@daml/types` (default -> namespace), strips `@daml/ledger` side-effect imports
+- **Package unification**: merges multiple DAR packages into a single entry point with named exports
+- **Self-reference resolution**: resolves `.d.ts` self-referencing imports (e.g. `@mypackage/codegen/daml-prim-DA-Types-1.0.0`) to local paths
+- **Version-aware registry**: generates a `versionedRegistry` function that conforms to `VersionedRegistry` from `@c7-digital/ledger`
+
+## Install
+
+```bash
+pnpm add -D @c7-digital/scribe
+```
+
+## Quick start
+
+### Zero-config
+
+Point scribe at the raw `dpm codegen-js` output:
+
+```bash
+pnpm exec scribe -i ./codegen/js
+```
+
+Scribe auto-detects:
+
+| What | How |
+|------|-----|
+| Main package | The non-`daml-prim`/`daml-stdlib`/`splice-*` package |
+| Modules | Every directory under `lib/` |
+| Templates & interfaces | Exports with `templateId` in `.d.ts` files |
+| Package ID | From `lib/index.d.ts` |
+| Version | From directory name (e.g. `my-project-0.1.0` -> `0.1.0`) |
+| Vendor packages | `splice-*` packages (third-party DARs your model depends on; exported separately from your main modules) |
+| Stdlib | `daml-prim-*`, `daml-stdlib-*`, `ghc-stdlib-*` |
+
+Output:
+
+```
+dist/
+├── codegen.js        # Bundled ESM (all modules + registry)
+├── index.js          # Re-exports from codegen.js
+├── index.d.ts        # TypeScript declarations
+├── version.js        # PACKAGE_VERSION export
+└── version.d.ts      # Version type declaration
+```
+
+### With config
+
+For projects that need aliases, selective exports, or compat versions, create a `scribe.yaml`:
+
+```yaml
+schema: "1.0"
+input: "../../codegen/js"
+
+main:
+  pattern: "domain-verification-model-*"
+
+vendor:
+  "splice-amulet-*":
+    modules:
+      "Splice.Amulet":
+        alias: Splice_Amulet
+        templates:
+          - FeaturedAppActivityMarker
+
+compat:
+  "domain-verification-model-*":
+    versions:
+      - hash: "8afd289e3ba826fcb23d955cfc108470b53527b0615745554185e5c9625b5832"
+        version: "0.0.5"
+```
+
+```bash
+pnpm exec scribe --config scribe.yaml
+```
+
+## CLI
+
+```
+scribe [options]
+
+Options:
+  -i, --input <path>     Path to codegen/js directory (default: ./codegen/js)
+  -c, --config <path>    Path to scribe.yaml config file
+  -v, --version <ver>    Override package version
+  -o, --output <path>    Output directory (default: dist)
+  --dry-run              Preview without writing files
+```
+
+**Version resolution priority**: `--version` flag > `$DAR_VERSION` env var > directory name.
+
+## Config reference
+
+All fields are optional except `schema`.
+
+```yaml
+schema: "1.0"                  # Required. Config schema version.
+
+input: "./codegen/js"          # Path to raw codegen output.
+                               # Default: ./codegen/js
+
+output:
+  dir: "dist"                  # Output directory. Default: dist
+  bundle: true                 # Run CJS->ESM bundling. Default: true
+
+main:
+  pattern: "my-project-*"     # Glob to identify the main package.
+                               # Only needed when auto-detection is ambiguous.
+
+  modules:                     # Restrict which modules are exported.
+                               # Omit to export all discovered modules.
+    MyModule:
+      templates:               # Restrict which templates are registered.
+        - MyTemplate           # Omit to register all.
+      interfaces:
+        - MyInterface
+      alias: CustomName        # Override the export name.
+    OtherModule: {}            # All templates/interfaces in this module.
+
+vendor:
+  "splice-amulet-*":          # Glob pattern matching vendor package names.
+    modules:
+      "Splice.Amulet":
+        alias: Splice_Amulet  # Export alias.
+        templates:
+          - FeaturedAppActivityMarker
+
+compat:
+  "my-project-*":              # Glob matching package names (main or vendor).
+    versions:                  # Backward-compat hashes for contract upgrades.
+      - hash: "8afd289e..."    # Package ID hash from a previous version.
+        version: "0.0.5"       # Version label for that hash.
+```
+
+**The config exists to constrain, not to describe.** If you omit `modules`, all are exported. If you omit `templates`, all are registered. Use config when you want *less* than the default, or need aliases and compat hashes.
+
+### Multi-version main packages
+
+When `codegen/js` contains multiple versions of the main package (e.g. `model-0.0.7`, `model-0.0.8`, `model-0.0.9`), scribe selects one:
+
+1. If `--version` or `$DAR_VERSION` is set, use that version
+2. Otherwise, pick the latest by semver
+
+Non-selected versions are demoted to stdlib (bundled as dependencies but not exported).
+
+### Vendor filtering
+
+When a `vendor` section is present, only `splice-*` packages matching a vendor pattern are included. Unmatched `splice-*` packages are excluded (demoted to stdlib). Without a `vendor` section, all `splice-*` packages are included as vendors.
+
+## Vite plugin
+
+Scribe ships a companion Vite plugin for consumer apps at `@c7-digital/scribe/vite`. This handles the runtime concerns that can't be solved at build time -- `@daml/types` and `@mojotech/json-type-validation` need special resolution in the consumer's dev server and build.
+
+```typescript
+import { defineConfig } from 'vite';
+import react from '@vitejs/plugin-react';
+import { damlCodegenPlugin } from '@c7-digital/scribe/vite';
+
+export default defineConfig({
+  plugins: [
+    damlCodegenPlugin(),
+    react(),
+  ],
+});
+```
+
+### What `damlCodegenPlugin()` configures
+
+- `@mojotech/json-type-validation` -> UMD build (required for ESM compat)
+- `@daml/types` -> local node_modules copy
+- `optimizeDeps` for `@daml/types`, `@c7-digital/ledger`, `@c7-digital/react`, and transitive deps
+
+### Options
+
+```typescript
+damlCodegenPlugin({
+  // Override node_modules lookup path.
+  // Useful in monorepos where node_modules isn't at cwd().
+  nodeModulesPath: __dirname,
+})
+```
+
+The plugin is independent of scribe's CLI -- it works for any app that imports `@daml/types`, even without using scribe for codegen.
+
+## Programmatic API
+
+```typescript
+import { run, loadConfig, discoverPackages, analyzePackage, generate } from '@c7-digital/scribe';
+
+// Run the full pipeline
+await run({ input: './codegen/js' });
+
+// Or use individual steps
+const config = await loadConfig({ config: './scribe.yaml' });
+const packages = await discoverPackages(config);
+const analyzed = await Promise.all(
+  packages.map(pkg => analyzePackage(pkg, config))
+);
+await generate(analyzed, config);
+```
+
+## Generated output
+
+### Exports
+
+Scribe generates named exports for each module and a flat `versionedRegistry`:
+
+```typescript
+// Main package modules
+import { InternetDomainName, AddressBook, C7Credentials } from '@mypackage/codegen';
+
+// Vendor modules (with aliases from config)
+import { Splice_Amulet } from '@mypackage/codegen';
+
+// Registry (conforms to VersionedRegistry from @c7-digital/ledger)
+import { versionedRegistry, PACKAGE_VERSION } from '@mypackage/codegen';
+```
+
+### Using with `@c7-digital/ledger`
+
+```typescript
+import { Ledger } from '@c7-digital/ledger';
+import { versionedRegistry, InternetDomainName } from '@mypackage/codegen';
+
+const ledger = new Ledger({
+  url: 'http://localhost:7575',
+  token: authToken,
+  versionedRegistry,  // plugs directly into ledger
+});
+
+const domains = await ledger.query(InternetDomainName.DomainOwnershipToken, {
+  owner: myParty,
+});
+```
+
+The `versionedRegistry` function is structurally compatible with `VersionedRegistry` from `@c7-digital/ledger` without a hard import dependency. TypeScript accepts it wherever `VersionedRegistry` is expected.
+
+## How it works
+
+### Pipeline
+
+1. **Load config** -- read `scribe.yaml` (if provided), merge with CLI flags and defaults
+2. **Discover** -- scan input directory, classify packages as main/vendor/stdlib
+3. **Analyze** -- extract `packageId`, modules, templates, and interfaces from each package's `.d.ts` files
+4. **Generate** -- produce `index.js` (imports, exports, registry), `index.d.ts`, `version.js`, `version.d.ts`
+5. **Bundle** -- Vite/Rollup build: CJS->ESM, import fixups, `registerTemplate` stripping, self-reference resolution
+
+### Architecture
+
+```
+dpm codegen-js          scribe                    consumer app
+┌─────────────┐    ┌──────────────────┐    ┌──────────────────────┐
+│ model-0.0.8/│    │ discover         │    │ vite.config.ts       │
+│ splice-*/   │--->│ analyze          │    │   damlCodegenPlugin()│
+│ daml-prim-*/│    │ generate + bundle│--->│                      │
+│ daml-stdlib/│    │                  │    │ import { ... }       │
+└─────────────┘    └──────────────────┘    │   from '@pkg/codegen'│
+                                           └──────────────────────┘
+```
+
+Scribe owns the **build** side (raw codegen -> clean ESM bundle). The `damlCodegenPlugin()` Vite plugin owns the **consumer** side (`@daml/types` resolution, optimizeDeps). Neither depends on the other.
+
+## Consumer `package.json` setup
+
+```json
+{
+  "name": "@mypackage/codegen",
+  "version": "0.0.1",
+  "type": "module",
+  "main": "dist/index.js",
+  "types": ".scribe/index.d.ts",
+  "exports": {
+    ".": {
+      "import": "./dist/index.js",
+      "types": "./.scribe/index.d.ts"
+    },
+    "./version": {
+      "import": "./dist/version.js",
+      "types": "./.scribe/version.d.ts"
+    }
+  },
+  "scripts": {
+    "build": "scribe --config scribe.yaml",
+    "prepare": "scribe --config scribe.yaml"
+  },
+  "dependencies": {
+    "@daml/types": "3.4.9",
+    "@mojotech/json-type-validation": "^3.1.0"
+  },
+  "devDependencies": {
+    "@c7-digital/scribe": "^0.0.1"
+  }
+}
+```
+
+Scribe creates a `.scribe/` staging directory (symlinked raw codegen packages + generated `.d.ts` files) and a `dist/` directory (bundled ESM). The `.scribe/` directory is tool-managed and should not be edited manually. The `types` field points at `.scribe/index.d.ts` so TypeScript resolves types through the original `.d.ts` chain, while `main`/`exports` point at the bundled `dist/` output for runtime.
+
+## Development
+
+### Testing
+
+```bash
+# Unit tests (fixture-based, no external dependencies)
+pnpm test
+
+# E2E tests (requires dpm or daml CLI)
+pnpm test:e2e
+```
+
+Unit tests validate scribe's transformation logic against hand-crafted fixtures in `src/__fixtures__/`. They're fast and don't require the Daml SDK.
+
+E2E tests validate the full pipeline against real codegen output: compile a Daml model, run `dpm codegen-js`, run scribe, then verify that a downstream consumer can type-check (`tsc --noEmit`) and bundle (`vite.build()`) the output. They require `dpm` (preferred) or the legacy `daml` assistant and will skip gracefully if neither is installed.
+
+### SDK version pin
+
+The Daml SDK version is pinned in two places:
+
+- `e2e/daml/daml.yaml` — `sdk-version` used to compile the test model
+- `.github/workflows/ci.yml` — version installed on CI runners
+
+Both must match `ledger`'s `@daml/types` dependency (currently **3.4.9**).
+
+## Requirements
+
+- Node.js >= 18.0.0
+- pnpm >= 8.0.0
+- Vite 5.x or 6.x (optional, for bundling and the companion plugin)
+- `dpm` or `daml` CLI (optional, for e2e tests only)

package/lib/src/analyze.d.ts

@@ -0,0 +1,22 @@
+import type { ResolvedConfig } from "./config.js";
+import type { DiscoveredPackage } from "./discover.js";
+export interface AnalyzedTemplate {
+    name: string;
+    kind: "template" | "interface";
+}
+export interface AnalyzedModule {
+    /** Module name, e.g. "InternetDomainName" */
+    name: string;
+    /** Relative path from package root to module directory */
+    modulePath: string;
+    /** Export alias (from config or auto-generated) */
+    alias: string;
+    /** Templates and interfaces found in this module */
+    members: AnalyzedTemplate[];
+}
+export interface AnalyzedPackage extends DiscoveredPackage {
+    packageId: string;
+    version: string;
+    modules: AnalyzedModule[];
+}
+export declare function analyzePackage(pkg: DiscoveredPackage, config: ResolvedConfig): Promise<AnalyzedPackage>;

package/lib/src/analyze.js

@@ -0,0 +1,175 @@
+import { readFile, readdir, stat } from "node:fs/promises";
+import { join } from "node:path";
+/**
+ * Extract packageId from lib/index.d.ts.
+ * Looks for: export declare const packageId = "...";
+ */
+async function extractPackageId(pkgPath) {
+    const indexDts = join(pkgPath, "lib", "index.d.ts");
+    const content = await readFile(indexDts, "utf-8");
+    const match = content.match(/export\s+declare\s+const\s+packageId\s*=\s*['"]([0-9a-f]+)['"]/);
+    if (!match?.[1]) {
+        throw new Error(`Could not extract packageId from ${indexDts}`);
+    }
+    return match[1];
+}
+/**
+ * Scan a module.d.ts file and classify exports as templates or interfaces.
+ *
+ * Templates have a `templateId` field and a `Template` type in their declaration.
+ * Interfaces have an `InterfaceCompanion` type.
+ */
+async function discoverMembers(moduleDtsPath) {
+    let content;
+    try {
+        content = await readFile(moduleDtsPath, "utf-8");
+    }
+    catch (err) {
+        const code = err.code;
+        if (code !== "ENOENT") {
+            console.warn(`Warning: could not read ${moduleDtsPath}: ${code ?? err}`);
+        }
+        return [];
+    }
+    const members = [];
+    // Match exported consts that look like template or interface companions.
+    // Pattern: export declare const Foo : ...Template<...>... or ...InterfaceCompanion<...>...
+    const regex = /export\s+declare\s+const\s+(\w+)\s*:\s*.*?(Template|InterfaceCompanion)\s*</g;
+    let m;
+    while ((m = regex.exec(content)) !== null) {
+        const name = m[1];
+        const kind = m[2] === "InterfaceCompanion" ? "interface" : "template";
+        members.push({ name, kind });
+    }
+    return members;
+}
+/**
+ * Discover modules inside a package's lib/ directory.
+ * Each subdirectory under lib/ that contains module.d.ts (or an index.d.ts
+ * for nested modules like Splice/Amulet/) is a module.
+ */
+async function discoverModules(pkgPath, role) {
+    const libPath = join(pkgPath, "lib");
+    const modules = [];
+    async function walk(dir, prefix) {
+        let entries;
+        try {
+            entries = await readdir(dir, { withFileTypes: true });
+        }
+        catch (err) {
+            const code = err.code;
+            if (code !== "ENOENT") {
+                console.warn(`Warning: could not read directory ${dir}: ${code ?? err}`);
+            }
+            return;
+        }
+        for (const entry of entries) {
+            if (!entry.isDirectory())
+                continue;
+            const subdir = join(dir, entry.name);
+            const moduleDts = join(subdir, "module.d.ts");
+            const indexDts = join(subdir, "index.d.ts");
+            // Check if this directory has module.d.ts (leaf module)
+            let dtsPath;
+            try {
+                await stat(moduleDts);
+                dtsPath = moduleDts;
+            }
+            catch {
+                try {
+                    await stat(indexDts);
+                    dtsPath = indexDts;
+                }
+                catch {
+                    // Not a leaf module; recurse
+                }
+            }
+            const moduleName = prefix ? `${prefix}.${entry.name}` : entry.name;
+            if (dtsPath) {
+                const members = await discoverMembers(dtsPath);
+                const alias = moduleName.replace(/\./g, "_");
+                modules.push({
+                    name: moduleName,
+                    modulePath: dtsPath.replace(libPath + "/", "").replace(/\.d\.ts$/, ""),
+                    alias,
+                    members,
+                });
+            }
+            // Always recurse for nested modules (e.g., Splice/Amulet/)
+            await walk(subdir, moduleName);
+        }
+    }
+    await walk(libPath, "");
+    return modules;
+}
+export async function analyzePackage(pkg, config) {
+    const packageId = await extractPackageId(pkg.path);
+    // Config version override only applies to the main package
+    const version = pkg.role === "main"
+        ? (config.version ?? pkg.detectedVersion ?? "0.0.0")
+        : (pkg.detectedVersion ?? "0.0.0");
+    let modules = await discoverModules(pkg.path, pkg.role);
+    // Apply config overrides for main package
+    if (pkg.role === "main" && config.main.modules) {
+        modules = applyModuleOverrides(modules, config.main.modules);
+    }
+    // Apply config overrides for vendor packages
+    if (pkg.role === "vendor" && config.vendor) {
+        const vendorOverride = findVendorOverride(pkg.name, config.vendor);
+        if (vendorOverride?.modules) {
+            modules = applyModuleOverrides(modules, vendorOverride.modules);
+        }
+    }
+    return {
+        ...pkg,
+        packageId,
+        version,
+        modules,
+    };
+}
+/**
+ * Apply config-driven overrides to discovered modules:
+ * - If overrides specify modules, only include those (filter)
+ * - Apply alias overrides
+ * - If overrides specify templates/interfaces, only include those (filter members)
+ */
+function applyModuleOverrides(discovered, overrides) {
+    const overrideKeys = Object.keys(overrides);
+    const result = [];
+    for (const key of overrideKeys) {
+        const override = overrides[key];
+        // Find discovered module matching this key (by name)
+        const mod = discovered.find((m) => m.name === key);
+        if (!mod) {
+            console.warn(`Config references module "${key}" but it was not found.`);
+            continue;
+        }
+        let members = mod.members;
+        // Filter templates if specified
+        if (override.templates) {
+            const allowedTemplates = new Set(override.templates);
+            members = members.filter((m) => m.kind !== "template" || allowedTemplates.has(m.name));
+        }
+        // Filter interfaces if specified
+        if (override.interfaces) {
+            const allowedInterfaces = new Set(override.interfaces);
+            members = members.filter((m) => m.kind !== "interface" || allowedInterfaces.has(m.name));
+        }
+        result.push({
+            ...mod,
+            alias: override.alias ?? mod.alias,
+            members,
+        });
+    }
+    return result;
+}
+/** Find the vendor override config entry matching a package name. */
+function findVendorOverride(pkgName, vendorConfig) {
+    for (const pattern of Object.keys(vendorConfig)) {
+        const regex = new RegExp("^" + pattern.replace(/\*/g, ".*") + "$");
+        if (regex.test(pkgName)) {
+            return vendorConfig[pattern];
+        }
+    }
+    return undefined;
+}

package/lib/src/cli.d.ts

@@ -0,0 +1,9 @@
+#!/usr/bin/env node
+export interface RunOptions {
+    input?: string;
+    config?: string;
+    version?: string;
+    output?: string;
+    dryRun?: boolean;
+}
+export declare function run(opts?: RunOptions): Promise<void>;

package/lib/src/cli.js

@@ -0,0 +1,35 @@
+#!/usr/bin/env node
+import { parseArgs } from "node:util";
+import { loadConfig } from "./config.js";
+import { discoverPackages } from "./discover.js";
+import { analyzePackage } from "./analyze.js";
+import { generate } from "./generate/index.js";
+export async function run(opts = {}) {
+    const config = await loadConfig(opts);
+    const packages = await discoverPackages(config);
+    const analyzed = await Promise.all(packages.map((pkg) => analyzePackage(pkg, config)));
+    await generate(analyzed, config);
+}
+async function main() {
+    const { values } = parseArgs({
+        options: {
+            input: { type: "string", short: "i" },
+            config: { type: "string", short: "c" },
+            version: { type: "string", short: "v" },
+            output: { type: "string", short: "o" },
+            "dry-run": { type: "boolean", default: false },
+        },
+        strict: true,
+    });
+    await run({
+        input: values.input,
+        config: values.config,
+        version: values.version,
+        output: values.output,
+        dryRun: values["dry-run"],
+    });
+}
+main().catch((err) => {
+    console.error(err);
+    process.exit(1);
+});