agentic-dataset-builder 0.2.0 → 0.2.2

package/README.md

@@ -1,20 +1,27 @@
 # Agentic Dataset Builder
 
-Pure TypeScript CLI for building one merged parquet dataset from local Pi, Codex, and Claude Code history.
+Pure TypeScript CLI for turning local Pi, Codex, and Claude Code history into one validated `dataset.parquet` file.
 
-## Requirements
+## Goal
 
-- Node 18+
+Use this repo when you want an AI coding assistant to do one job end-to-end:
 
-## Install and run
+1. discover local session history
+2. normalize it into the local Qwen35-compatible schema
+3. label records by training use
+4. write one final parquet dataset
 
-Without installing globally:
+The CLI is native Node.js + TypeScript. It does not require Python.
+
+## Fastest path
+
+If the package is published on npm:
 
 ```bash
-npx agentic-dataset-builder@0.2.0 --output-root ./out
+npx --registry=https://registry.npmjs.org/ agentic-dataset-builder@0.2.2 --output-root ./out
 ```
 
-Local development:
+If working from this repo locally:
 
 ```bash
 npm install
@@ -22,42 +29,126 @@ npm run build
 node dist/cli.js --output-root ./out
 ```
 
-## Examples
+## What the command does
 
-```bash
-# Pi + Codex
-npx agentic-dataset-builder@0.2.0 --output-root ./out --include-sources pi,codex --include-labels cot_eligible,agent_only
+The CLI will:
 
-# Codex + Claude prompt-only
-npx agentic-dataset-builder@0.2.0 --output-root ./out --include-sources codex,claude --include-labels agent_only,prompt_only
+- detect local session roots for `pi`, `codex`, and `claude`
+- read supported history files
+- validate normalized records with `Zod`
+- keep only the labels you requested
+- write one final parquet file
+- write a manifest and a run log
 
-# Pi only
-npx agentic-dataset-builder@0.2.0 --output-root ./out --include-sources pi --include-labels cot_eligible,agent_only
-```
+## Default source behavior
+
+- `pi`
+  - full agent traces
+  - can produce `cot_eligible` or `agent_only`
+- `codex`
+  - full agent traces
+  - usually produces `agent_only`
+- `claude`
+  - prompt history only for now
+  - produces `prompt_only`
 
-## Output
+Claude is intentionally low-fidelity right now. It is not treated as a full assistant/tool trace source.
 
-Each run creates a directory like:
+## Default output
+
+Each run creates one directory:
 
 ```text
-out/agentic-dataset-<timestamp>/
+<output-root>/agentic-dataset-<timestamp>/
   dataset.parquet
   manifest.json
   run.log
 ```
 
-- `dataset.parquet`: final merged dataset
-- `manifest.json`: source roots, counts, labels, and summary stats
-- `run.log`: step-by-step execution log
+Files:
+
+- `dataset.parquet`
+  - final merged dataset
+- `manifest.json`
+  - source roots, source counts, labels kept, output path
+- `run.log`
+  - step-by-step execution log for debugging
+
+## Recommended commands
+
+Pi + Codex:
+
+```bash
+node dist/cli.js --output-root ./out --include-sources pi,codex --include-labels cot_eligible,agent_only
+```
+
+Codex + Claude prompt-only:
+
+```bash
+node dist/cli.js --output-root ./out --include-sources codex,claude --include-labels agent_only,prompt_only
+```
+
+Pi only:
+
+```bash
+node dist/cli.js --output-root ./out --include-sources pi --include-labels cot_eligible,agent_only
+```
+
+## Important flags
+
+- `--output-root <dir>`
+  - required output root
+- `--include-sources <csv>`
+  - any of: `pi,codex,claude`
+- `--include-labels <csv>`
+  - any of: `cot_eligible,agent_only,prompt_only,discard`
+- `--pi-root <dir>`
+  - override detected Pi session path
+- `--codex-root <dir>`
+  - override detected Codex session path
+- `--claude-root <dir>`
+  - override detected Claude project-history path
+- `--help`
+  - print CLI help
+
+## Auto-detected paths
+
+The CLI tries OS-specific defaults automatically.
+
+Typical paths:
+
+- Pi: `~/.pi/agent/sessions`
+- Codex: `~/.codex/sessions`
+- Claude: `~/.claude/projects`
+
+On Windows it also checks `APPDATA` and `LOCALAPPDATA` variants.
+
+## Verification checklist
+
+After a run, verify these three things:
 
-## Source support
+1. `dataset.parquet` exists
+2. `manifest.json` exists
+3. `run.log` does not end with an uncaught error
 
-- `pi`: full agent trace with visible reasoning when available
-- `codex`: agent trace, often without visible reasoning
-- `claude`: prompt-history only for now, labeled `prompt_only`
+Typical quick check:
+
+```bash
+ls ./out/agentic-dataset-*/
+```
+
+## Development notes
+
+Useful development commands:
+
+```bash
+npm run check
+npm run test
+npm run build
+```
 
-## Notes
+This repo currently includes:
 
-- default source roots are auto-detected for Linux, macOS, and Windows
-- override paths with `--pi-root`, `--codex-root`, and `--claude-root`
-- Claude is intentionally low-fidelity right now: user prompt history only, not full assistant/tool trace
+- Zod validation for source events and final records
+- Vitest coverage for core schema and labeling paths
+- native parquet writing in TypeScript

package/dist/cli.js

@@ -9,6 +9,23 @@ import { labelRecord } from './labeling.js';
 import { Qwen35RecordSchema } from './schemas/qwen35.js';
 import { writeParquet } from './parquet.js';
 function parseArgs(argv) {
+    if (argv.includes('--help') || argv.includes('-h')) {
+        console.log(`agentic-dataset-builder@0.2.2
+
+Usage:
+  npx agentic-dataset-builder@0.2.2 --output-root ./out
+
+Options:
+  --output-root <dir>       Output directory root
+  --include-sources <list>  Comma-separated: pi,codex,claude
+  --include-labels <list>   Comma-separated: cot_eligible,agent_only,prompt_only,discard
+  --pi-root <dir>           Override Pi session root
+  --codex-root <dir>        Override Codex session root
+  --claude-root <dir>       Override Claude project history root
+  --help                    Show this help message
+`);
+        process.exit(0);
+    }
     const args = {
         outputRoot: './out',
         includeSources: ['pi', 'codex'],

package/dist/platform/paths.js

@@ -1,13 +1,36 @@
 import os from 'node:os';
 import path from 'node:path';
 import fs from 'node:fs';
+function currentPlatform() {
+    return process.platform;
+}
+function xdgConfigHome() {
+    return process.env.XDG_CONFIG_HOME;
+}
+function xdgDataHome() {
+    return process.env.XDG_DATA_HOME;
+}
+function macAppSupportHome() {
+    if (currentPlatform() !== 'darwin')
+        return undefined;
+    return path.join(os.homedir(), 'Library', 'Application Support');
+}
 export function candidatePiRoots() {
     const home = os.homedir();
     const appdata = process.env.APPDATA;
     const localappdata = process.env.LOCALAPPDATA;
+    const xdgConfig = xdgConfigHome();
+    const xdgData = xdgDataHome();
+    const macAppSupport = macAppSupportHome();
     return dedupe([
         process.env.PI_SESSION_ROOT,
         path.join(home, '.pi', 'agent', 'sessions'),
+        xdgConfig ? path.join(xdgConfig, 'pi', 'agent', 'sessions') : undefined,
+        xdgConfig ? path.join(xdgConfig, '.pi', 'agent', 'sessions') : undefined,
+        xdgData ? path.join(xdgData, 'pi', 'agent', 'sessions') : undefined,
+        xdgData ? path.join(xdgData, '.pi', 'agent', 'sessions') : undefined,
+        macAppSupport ? path.join(macAppSupport, 'pi', 'agent', 'sessions') : undefined,
+        macAppSupport ? path.join(macAppSupport, '.pi', 'agent', 'sessions') : undefined,
         appdata ? path.join(appdata, 'pi', 'agent', 'sessions') : undefined,
         appdata ? path.join(appdata, '.pi', 'agent', 'sessions') : undefined,
         localappdata ? path.join(localappdata, 'pi', 'agent', 'sessions') : undefined,
@@ -18,9 +41,18 @@ export function candidateCodexRoots() {
     const home = os.homedir();
     const appdata = process.env.APPDATA;
     const localappdata = process.env.LOCALAPPDATA;
+    const xdgConfig = xdgConfigHome();
+    const xdgData = xdgDataHome();
+    const macAppSupport = macAppSupportHome();
     return dedupe([
         process.env.CODEX_SESSION_ROOT,
         path.join(home, '.codex', 'sessions'),
+        xdgConfig ? path.join(xdgConfig, 'codex', 'sessions') : undefined,
+        xdgConfig ? path.join(xdgConfig, '.codex', 'sessions') : undefined,
+        xdgData ? path.join(xdgData, 'codex', 'sessions') : undefined,
+        xdgData ? path.join(xdgData, '.codex', 'sessions') : undefined,
+        macAppSupport ? path.join(macAppSupport, 'Codex', 'sessions') : undefined,
+        macAppSupport ? path.join(macAppSupport, '.codex', 'sessions') : undefined,
         appdata ? path.join(appdata, 'Codex', 'sessions') : undefined,
         appdata ? path.join(appdata, '.codex', 'sessions') : undefined,
         localappdata ? path.join(localappdata, 'Codex', 'sessions') : undefined,
@@ -31,9 +63,18 @@ export function candidateClaudeRoots() {
     const home = os.homedir();
     const appdata = process.env.APPDATA;
     const localappdata = process.env.LOCALAPPDATA;
+    const xdgConfig = xdgConfigHome();
+    const xdgData = xdgDataHome();
+    const macAppSupport = macAppSupportHome();
     return dedupe([
         process.env.CLAUDE_SESSION_ROOT,
         path.join(home, '.claude', 'projects'),
+        xdgConfig ? path.join(xdgConfig, 'claude', 'projects') : undefined,
+        xdgConfig ? path.join(xdgConfig, '.claude', 'projects') : undefined,
+        xdgData ? path.join(xdgData, 'claude', 'projects') : undefined,
+        xdgData ? path.join(xdgData, '.claude', 'projects') : undefined,
+        macAppSupport ? path.join(macAppSupport, 'Claude', 'projects') : undefined,
+        macAppSupport ? path.join(macAppSupport, '.claude', 'projects') : undefined,
         appdata ? path.join(appdata, 'Claude', 'projects') : undefined,
         appdata ? path.join(appdata, '.claude', 'projects') : undefined,
         localappdata ? path.join(localappdata, 'Claude', 'projects') : undefined,

package/dist/platform/paths.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/platform/paths.test.js

@@ -0,0 +1,36 @@
+import { describe, expect, it, vi, beforeEach, afterEach } from 'vitest';
+const envBackup = { ...process.env };
+describe('platform path candidates', () => {
+    beforeEach(() => {
+        vi.resetModules();
+        process.env = { ...envBackup };
+    });
+    afterEach(() => {
+        process.env = { ...envBackup };
+        vi.unstubAllGlobals();
+    });
+    it('includes XDG candidates for pi on linux', async () => {
+        process.env.XDG_CONFIG_HOME = '/home/test/.config';
+        process.env.XDG_DATA_HOME = '/home/test/.local/share';
+        vi.stubGlobal('process', { ...process, platform: 'linux', env: process.env });
+        const mod = await import('./paths.js');
+        const candidates = mod.candidatePiRoots();
+        expect(candidates).toContain('/home/test/.config/pi/agent/sessions');
+        expect(candidates).toContain('/home/test/.local/share/pi/agent/sessions');
+    });
+    it('includes Application Support candidates for codex on macOS', async () => {
+        vi.stubGlobal('process', { ...process, platform: 'darwin', env: process.env });
+        const mod = await import('./paths.js');
+        const candidates = mod.candidateCodexRoots();
+        expect(candidates.some((value) => value.includes('Library/Application Support/Codex/sessions'))).toBe(true);
+    });
+    it('includes APPDATA and LOCALAPPDATA candidates for claude on Windows', async () => {
+        process.env.APPDATA = 'C:/Users/test/AppData/Roaming';
+        process.env.LOCALAPPDATA = 'C:/Users/test/AppData/Local';
+        vi.stubGlobal('process', { ...process, platform: 'win32', env: process.env });
+        const mod = await import('./paths.js');
+        const candidates = mod.candidateClaudeRoots();
+        expect(candidates).toContain('C:/Users/test/AppData/Roaming/Claude/projects');
+        expect(candidates).toContain('C:/Users/test/AppData/Local/Claude/projects');
+    });
+});

package/dist/schemas/source.d.ts

@@ -0,0 +1,23 @@
+import { z } from 'zod';
+export declare const PiSessionHeaderSchema: z.ZodObject<{
+    type: z.ZodLiteral<"session">;
+    id: z.ZodString;
+    timestamp: z.ZodString;
+    cwd: z.ZodOptional<z.ZodString>;
+}, z.core.$loose>;
+export declare const PiSessionEntrySchema: z.ZodObject<{
+    type: z.ZodString;
+    id: z.ZodOptional<z.ZodString>;
+    parentId: z.ZodOptional<z.ZodNullable<z.ZodString>>;
+    timestamp: z.ZodOptional<z.ZodString>;
+}, z.core.$loose>;
+export declare const CodexEntrySchema: z.ZodObject<{
+    timestamp: z.ZodOptional<z.ZodString>;
+    type: z.ZodString;
+    payload: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+}, z.core.$loose>;
+export declare const ClaudeProjectEntrySchema: z.ZodRecord<z.ZodString, z.ZodUnknown>;
+export type PiSessionHeader = z.infer<typeof PiSessionHeaderSchema>;
+export type PiSessionEntry = z.infer<typeof PiSessionEntrySchema>;
+export type CodexEntry = z.infer<typeof CodexEntrySchema>;
+export type ClaudeProjectEntry = z.infer<typeof ClaudeProjectEntrySchema>;

package/dist/schemas/source.js

@@ -0,0 +1,19 @@
+import { z } from 'zod';
+export const PiSessionHeaderSchema = z.object({
+    type: z.literal('session'),
+    id: z.string(),
+    timestamp: z.string(),
+    cwd: z.string().optional(),
+}).passthrough();
+export const PiSessionEntrySchema = z.object({
+    type: z.string(),
+    id: z.string().optional(),
+    parentId: z.string().nullable().optional(),
+    timestamp: z.string().optional(),
+}).passthrough();
+export const CodexEntrySchema = z.object({
+    timestamp: z.string().optional(),
+    type: z.string(),
+    payload: z.record(z.string(), z.unknown()).optional(),
+}).passthrough();
+export const ClaudeProjectEntrySchema = z.record(z.string(), z.unknown());

package/dist/sources/claude.js

@@ -1,13 +1,12 @@
 import fg from 'fast-glob';
-import { z } from 'zod';
 import { Qwen35RecordSchema } from '../schemas/qwen35.js';
+import { ClaudeProjectEntrySchema } from '../schemas/source.js';
 import { readJsonl } from '../utils/jsonl.js';
-const EntrySchema = z.record(z.string(), z.unknown());
 export async function collectClaudePromptOnlyRecords(root) {
     const files = await fg('**/*.jsonl', { cwd: root, absolute: true, onlyFiles: true });
     const records = [];
     for (const file of files.sort()) {
-        const entries = (await readJsonl(file)).map((row) => EntrySchema.parse(row));
+        const entries = (await readJsonl(file)).map((row) => ClaudeProjectEntrySchema.parse(row));
         for (const entry of entries) {
             if (entry.type !== 'user')
                 continue;

package/dist/sources/codex.js

@@ -1,12 +1,7 @@
 import fg from 'fast-glob';
-import { z } from 'zod';
 import { Qwen35RecordSchema } from '../schemas/qwen35.js';
+import { CodexEntrySchema } from '../schemas/source.js';
 import { readJsonl } from '../utils/jsonl.js';
-const EntrySchema = z.object({
-    timestamp: z.string().optional(),
-    type: z.string(),
-    payload: z.record(z.string(), z.unknown()).optional(),
-}).passthrough();
 class TurnBuilder {
     sessionMeta;
     turnId;
@@ -204,7 +199,7 @@ export async function collectCodexRecords(root) {
     const files = await fg('**/*.jsonl', { cwd: root, absolute: true, onlyFiles: true });
     const records = [];
     for (const file of files.sort()) {
-        const entries = (await readJsonl(file)).map((entry) => EntrySchema.parse(entry));
+        const entries = (await readJsonl(file)).map((entry) => CodexEntrySchema.parse(entry));
         const sessionMeta = (entries.find((entry) => entry.type === 'session_meta')?.payload ?? {});
         let builder = null;
         for (const entry of entries) {

package/dist/sources/pi.js

@@ -1,20 +1,15 @@
 import fs from 'node:fs';
 import fg from 'fast-glob';
-import { z } from 'zod';
 import { Qwen35RecordSchema } from '../schemas/qwen35.js';
+import { PiSessionEntrySchema, PiSessionHeaderSchema } from '../schemas/source.js';
 import { isFile } from '../utils/common.js';
 import { readJsonl } from '../utils/jsonl.js';
-const SessionEntrySchema = z.object({
-    type: z.string(),
-    id: z.string().optional(),
-    parentId: z.string().nullable().optional(),
-    timestamp: z.string().optional(),
-}).passthrough();
 export async function collectPiRecords(root) {
     const files = await fg('**/*.jsonl', { cwd: root, absolute: true, onlyFiles: true });
     const records = [];
     for (const file of files.sort()) {
-        const rows = (await readJsonl(file)).map((row) => SessionEntrySchema.parse(row));
+        const rawRows = await readJsonl(file);
+        const rows = rawRows.map((row, index) => (index === 0 ? PiSessionHeaderSchema.parse(row) : PiSessionEntrySchema.parse(row)));
         if (!rows.length)
             continue;
         const header = rows[0];
@@ -25,7 +20,7 @@ export async function collectPiRecords(root) {
             if (!entry.id)
                 continue;
             byId.set(entry.id, entry);
-            const key = entry.parentId ?? null;
+            const key = typeof entry.parentId === 'string' ? entry.parentId : null;
             const bucket = children.get(key) ?? [];
             bucket.push(entry.id);
             children.set(key, bucket);
@@ -47,7 +42,7 @@ function branchEntries(leaf, byId) {
         if (!entry)
             break;
         ordered.push(entry);
-        current = entry.parentId ?? null;
+        current = typeof entry.parentId === 'string' ? entry.parentId : null;
     }
     return ordered.reverse();
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "agentic-dataset-builder",
-  "version": "0.2.0",
+  "version": "0.2.2",
   "description": "Pure TypeScript agentic dataset builder for Pi, Codex, and Claude Code history",
   "license": "MIT",
   "type": "module",
@@ -26,7 +26,24 @@
     "agentic-dataset-builder": "./dist/cli.js"
   },
   "files": [
-    "dist",
+    "dist/cli.js",
+    "dist/cli.d.ts",
+    "dist/labeling.js",
+    "dist/labeling.d.ts",
+    "dist/parquet.js",
+    "dist/parquet.d.ts",
+    "dist/platform",
+    "dist/schemas/qwen35.js",
+    "dist/schemas/qwen35.d.ts",
+    "dist/schemas/source.js",
+    "dist/schemas/source.d.ts",
+    "dist/sources/pi.js",
+    "dist/sources/pi.d.ts",
+    "dist/sources/codex.js",
+    "dist/sources/codex.d.ts",
+    "dist/sources/claude.js",
+    "dist/sources/claude.d.ts",
+    "dist/utils",
     "README.md",
     "LICENSE"
   ],
@@ -37,7 +54,8 @@
     "build": "tsc -p tsconfig.json",
     "dev": "tsx src/cli.ts",
     "check": "tsc -p tsconfig.json --noEmit",
-    "pack:check": "npm pack --dry-run"
+    "pack:check": "npm pack --dry-run",
+    "test": "vitest run"
   },
   "dependencies": {
     "fast-glob": "^3.3.3",
@@ -47,6 +65,7 @@
   "devDependencies": {
     "@types/node": "^24.7.2",
     "tsx": "^4.20.6",
-    "typescript": "^5.9.3"
+    "typescript": "^5.9.3",
+    "vitest": "^4.1.3"
   }
 }