forge-workflow 0.0.9 → 0.1.0-beta.2

package/lib/kernel/planning-buckets-schema.js

@@ -0,0 +1,109 @@
+'use strict';
+
+const { cloneTableList, deepFreeze, field, index, table, validateKernelSchema } = require('./schema');
+
+// Planning buckets are first-class entities, not string fields on issues (D5/D18,
+// forge-2agy.9.2.7). Sprint/release/milestone each carry id, name, state, dates,
+// owner/goal, ordering rank, completion rollups, and an entity revision.
+const SPRINT_STATES = Object.freeze(['planned', 'active', 'completed', 'cancelled']);
+const RELEASE_STATES = Object.freeze(['planned', 'in_progress', 'released', 'cancelled']);
+const MILESTONE_STATES = Object.freeze(['planned', 'reached', 'missed', 'cancelled']);
+
+// Board drag/drop and assignment operations emit Kernel events carrying
+// expected_revision + idempotency_key (board rank / mutation event model,
+// forge-2agy.9.2.6). See docs/reference/KERNEL_TAXONOMY_VALIDATION.md.
+const BOARD_MUTATION_EVENT_TYPES = Object.freeze([
+	'issue.reordered',
+	'issue.status_changed',
+	'issue.sprint_assigned',
+	'issue.release_assigned',
+	'issue.blocked',
+	'issue.unblocked',
+	'issue.type_changed',
+]);
+
+const ROLLUP = Object.freeze({ storageClass: 'read_model' });
+
+// All three planning buckets share the same envelope (identity, state, ownership,
+// ordering rank, completion rollups, and revision). `specificFields` carries the
+// bucket-specific scheduling columns; this keeps the shared shape defined once.
+function planningBucketTable(name, specificFields, indexes) {
+	return table(name, 'authority', [
+		field('id', 'TEXT', { primaryKey: true }),
+		field('name', 'TEXT', { notNull: true }),
+		field('state', 'TEXT', { notNull: true, default: "'planned'" }),
+		field('goal', 'TEXT'),
+		field('owner', 'TEXT'),
+		...specificFields,
+		field('rank', 'INTEGER', { notNull: true, default: '0' }),
+		field('total_count', 'INTEGER', { notNull: true, default: '0', ...ROLLUP }),
+		field('completed_count', 'INTEGER', { notNull: true, default: '0', ...ROLLUP }),
+		field('created_at', 'TEXT', { notNull: true }),
+		field('updated_at', 'TEXT', { notNull: true }),
+		field('entity_revision', 'INTEGER', { notNull: true, default: '0' }),
+	], indexes);
+}
+
+const PLANNING_BUCKET_TABLE_LIST = deepFreeze([
+	planningBucketTable('sprint', [
+		field('start_date', 'TEXT'),
+		field('end_date', 'TEXT'),
+		field('capacity', 'INTEGER'),
+	], [
+		index('idx_kernel_sprint_state', ['state']),
+		index('idx_kernel_sprint_rank', ['rank']),
+	]),
+	planningBucketTable('release', [
+		field('target_date', 'TEXT'),
+		field('released_at', 'TEXT'),
+	], [
+		index('idx_kernel_release_state', ['state']),
+		index('idx_kernel_release_rank', ['rank']),
+	]),
+	planningBucketTable('milestone', [
+		field('release_id', 'TEXT'),
+		field('target_date', 'TEXT'),
+		field('reached_at', 'TEXT'),
+	], [
+		index('idx_kernel_milestone_state', ['state']),
+		index('idx_kernel_milestone_release', ['release_id']),
+	]),
+]);
+
+const PLANNING_BUCKET_TABLES = deepFreeze(
+	Object.fromEntries(PLANNING_BUCKET_TABLE_LIST.map(definition => [definition.name, definition])),
+);
+
+/** @returns {{version: number, tables: object[]}} an isolated, deep-cloned copy of the planning-bucket schema. */
+function getPlanningBucketsSchema() {
+	return {
+		version: 1,
+		tables: cloneTableList(PLANNING_BUCKET_TABLE_LIST),
+	};
+}
+
+/**
+ * Validate the planning-bucket schema: enforce that the sprint/release/milestone tables are
+ * present, then run the shared Kernel structural validator.
+ * @throws {Error} if a required planning bucket table is missing.
+ * @returns {true} when membership, storage classes, and authorities are valid.
+ */
+function validatePlanningBucketsSchema(schema = getPlanningBucketsSchema()) {
+	const present = new Set((schema.tables || []).map(tableDefinition => tableDefinition.name));
+	for (const required of ['sprint', 'release', 'milestone']) {
+		if (!present.has(required)) {
+			throw new Error(`Missing planning bucket table: ${required}`);
+		}
+	}
+	return validateKernelSchema(schema);
+}
+
+module.exports = {
+	BOARD_MUTATION_EVENT_TYPES,
+	MILESTONE_STATES,
+	PLANNING_BUCKET_TABLES,
+	RELEASE_STATES,
+	SPRINT_STATES,
+	getPlanningBucketsSchema,
+	validatePlanningBucketsSchema,
+};

package/lib/kernel/projection-jsonl-writer.js

@@ -0,0 +1,450 @@
+'use strict';
+
+const crypto = require('node:crypto');
+const fs = require('node:fs');
+const path = require('node:path');
+const { parseJsonl, stringifyJsonl } = require('../adapters/beads-kernel-compat');
+
+// Schema history (all additive; importProjection accepts any schema_version <= this):
+//   v1 — the original 11-key issue shape.
+//   v2 — added `created_by` (v1 lost the author on round-trip).
+//   v3 — carries the FULL kernel_issues column set (parent_id, sprint_id, release_id,
+//        stage_state, labels, acceptance_criteria, estimate, design, notes, assignee,
+//        closed_at, close_reason, metadata) so export → hydrate is lossless. v1/v2
+//        snapshots still import; their newer columns are simply absent → null.
+const SCHEMA_VERSION = 3;
+
+// Deterministic write/rename order. manifest.json is renamed last so a partial
+// write never leaves a manifest pointing at incomplete JSONL (the manifest is
+// effectively the commit marker for the snapshot).
+const PROJECTION_FILE_ORDER = ['issues.jsonl', 'comments.jsonl', 'dependencies.jsonl', 'manifest.json'];
+const DEFAULT_PROJECTION_DIR = path.join('.forge', 'kernel');
+// Exclusive publish lock filename (transient — written and removed within a single
+// writeProjection call) that serializes the rename phase across concurrent exports.
+const PROJECTION_LOCK_FILE = '.export.lock';
+
+// Fixed key insertion order (v3): 'kind' + the full kernel_issues column set, in the
+// table's column order, so export → hydrate round-trips every column losslessly.
+const ISSUE_KEYS = [
+	'kind',
+	'id', 'title', 'body', 'type', 'status', 'priority', 'priority_rank',
+	'created_at', 'updated_at', 'entity_revision',
+	'parent_id', 'sprint_id', 'release_id', 'stage_state', 'labels',
+	'acceptance_criteria', 'estimate', 'design', 'notes', 'assignee',
+	'created_by', 'closed_at', 'close_reason', 'metadata',
+];
+const COMMENT_KEYS = ['kind', 'id', 'issue_id', 'body', 'actor', 'visibility', 'created_at'];
+const DEP_KEYS = ['kind', 'id', 'issue_id', 'blocks_issue_id', 'dependency_type', 'created_at'];
+
+function pickKeys(record, keys) {
+	const out = {};
+	for (const key of keys) {
+		out[key] = Object.prototype.hasOwnProperty.call(record, key) ? record[key] : null;
+	}
+	return out;
+}
+
+function normalizeIssue(raw) {
+	return pickKeys({ ...raw, kind: 'issue', body: raw.body ?? null }, ISSUE_KEYS);
+}
+
+function normalizeComment(raw) {
+	return pickKeys({ ...raw, kind: 'comment' }, COMMENT_KEYS);
+}
+
+function normalizeDependency(raw) {
+	return pickKeys({ ...raw, kind: 'dependency' }, DEP_KEYS);
+}
+
+function normalizeProjectionModel(model) {
+	return {
+		issues: (model.issues || []).map(normalizeIssue),
+		comments: (model.comments || []).map(normalizeComment),
+		dependencies: (model.dependencies || []).map(normalizeDependency),
+	};
+}
+
+function cmp(a, b) {
+	if (a < b) return -1;
+	if (a > b) return 1;
+	return 0;
+}
+
+function buildProjectionSnapshot(model) {
+	const issues = [...model.issues].sort((a, b) => cmp(a.id, b.id));
+
+	const comments = [...model.comments].sort((a, b) =>
+		cmp(a.issue_id, b.issue_id) || cmp(a.created_at, b.created_at) || cmp(a.id, b.id),
+	);
+
+	const dependencies = [...model.dependencies].sort((a, b) =>
+		cmp(a.issue_id, b.issue_id) ||
+		cmp(a.blocks_issue_id, b.blocks_issue_id) ||
+		cmp(a.dependency_type, b.dependency_type) ||
+		cmp(a.id, b.id),
+	);
+
+	return { issues, comments, dependencies };
+}
+
+function serializeProjection(model) {
+	// Normalize defensively: callers (including the outbox consumer) may pass raw
+	// driver rows that lack `kind`/canonical key order. normalizeProjectionModel is
+	// idempotent, so pre-normalized snapshots are unaffected.
+	const snap = buildProjectionSnapshot(normalizeProjectionModel(model));
+
+	const issuesJsonl = stringifyJsonl(snap.issues);
+	const commentsJsonl = stringifyJsonl(snap.comments);
+	const depsJsonl = stringifyJsonl(snap.dependencies);
+
+	const contentSha256 = crypto
+		.createHash('sha256')
+		.update(issuesJsonl)
+		.update(commentsJsonl)
+		.update(depsJsonl)
+		.digest('hex');
+
+	const manifest = {
+		schema_version: SCHEMA_VERSION,
+		source: 'kernel',
+		counts: {
+			issues: snap.issues.length,
+			comments: snap.comments.length,
+			dependencies: snap.dependencies.length,
+		},
+		content_sha256: contentSha256,
+	};
+
+	return {
+		files: {
+			'issues.jsonl': issuesJsonl,
+			'comments.jsonl': commentsJsonl,
+			'dependencies.jsonl': depsJsonl,
+			'manifest.json': JSON.stringify(manifest, null, 2) + '\n',
+		},
+	};
+}
+
+function importProjection(files) {
+	if (!files['manifest.json']) {
+		throw new Error('Missing manifest.json in projection files');
+	}
+
+	const manifest = JSON.parse(files['manifest.json']);
+
+	// Versioned-manifest guard: refuse a snapshot written by a NEWER forge whose
+	// format we do not understand, rather than silently mis-importing it. Older
+	// versions (<= ours) remain importable — missing fields default on write.
+	const schemaVersion = manifest.schema_version;
+	if (!Number.isInteger(schemaVersion) || schemaVersion < 1) {
+		throw new Error(
+			`Invalid projection manifest: schema_version must be a positive integer, got ${JSON.stringify(schemaVersion)}`,
+		);
+	}
+	if (schemaVersion > SCHEMA_VERSION) {
+		throw new Error(
+			`Unsupported projection schema_version ${schemaVersion}: this forge understands up to ${SCHEMA_VERSION}. `
+			+ 'Upgrade forge to import this snapshot.',
+		);
+	}
+
+	const issuesJsonl = files['issues.jsonl'] || '';
+	const commentsJsonl = files['comments.jsonl'] || '';
+	const depsJsonl = files['dependencies.jsonl'] || '';
+
+	const actualSha256 = crypto
+		.createHash('sha256')
+		.update(issuesJsonl)
+		.update(commentsJsonl)
+		.update(depsJsonl)
+		.digest('hex');
+
+	if (actualSha256 !== manifest.content_sha256) {
+		throw new Error(
+			`sha256 mismatch: manifest has ${manifest.content_sha256}, computed ${actualSha256}`,
+		);
+	}
+
+	const issues = issuesJsonl ? parseJsonl(issuesJsonl, 'issues.jsonl') : [];
+	const comments = commentsJsonl ? parseJsonl(commentsJsonl, 'comments.jsonl') : [];
+	const dependencies = depsJsonl ? parseJsonl(depsJsonl, 'dependencies.jsonl') : [];
+
+	// The content hash binds only the concatenated byte stream, not file
+	// boundaries — moving a line between files leaves the stream unchanged. Bind
+	// records to their files (kind) and to the manifest (counts) so a relocated or
+	// dropped record is rejected even when the hash still matches.
+	assertRecordKind(issues, 'issue', 'issues.jsonl');
+	assertRecordKind(comments, 'comment', 'comments.jsonl');
+	assertRecordKind(dependencies, 'dependency', 'dependencies.jsonl');
+	assertManifestCounts(manifest.counts, { issues, comments, dependencies });
+
+	return { issues, comments, dependencies };
+}
+
+function assertRecordKind(records, expectedKind, file) {
+	for (let i = 0; i < records.length; i += 1) {
+		if (records[i].kind !== expectedKind) {
+			throw new Error(
+				`Invalid record in ${file} at line ${i + 1}: kind "${records[i].kind}", expected "${expectedKind}"`,
+			);
+		}
+	}
+}
+
+function assertManifestCounts(counts, parsed) {
+	if (!counts) {
+		// A manifest with a valid hash but no counts would otherwise skip the
+		// per-file binding check entirely — treat the missing field as corruption.
+		throw new Error('Manifest is missing the required counts field');
+	}
+	for (const key of ['issues', 'comments', 'dependencies']) {
+		if (typeof counts[key] === 'number' && counts[key] !== parsed[key].length) {
+			throw new Error(
+				`Manifest count mismatch for ${key}: manifest says ${counts[key]}, found ${parsed[key].length}`,
+			);
+		}
+	}
+}
+
+function resolveProjectionDir(projectionDir, projectRoot) {
+	const resolved = path.resolve(projectionDir || (projectRoot ? path.join(projectRoot, DEFAULT_PROJECTION_DIR) : DEFAULT_PROJECTION_DIR));
+
+	if (projectRoot) {
+		const root = path.resolve(projectRoot);
+		const relative = path.relative(root, resolved);
+		const escapes = relative.startsWith('..') || path.isAbsolute(relative);
+		if (escapes) {
+			throw new Error(`Projection directory escapes the project root: ${resolved}`);
+		}
+	}
+
+	return resolved;
+}
+
+// Atomic-ish multi-file write: render every file to a sibling temp, snapshot the
+// existing targets, then rename temps into place in a fixed order. If any rename
+// throws, restore the snapshot (rewrite prior content or delete files that did
+// not previously exist) and clean up temps before rethrowing. A projection write
+// failure must never leave a half-applied snapshot on disk.
+//
+// The per-file renames are not individually atomic across files, so the publish
+// phase is guarded by an exclusive directory lock: a concurrent export to the
+// same dir fails fast instead of interleaving renames into a mixed snapshot whose
+// manifest hash no longer matches. The lock is always released in `finally`, so
+// only an abnormal crash mid-publish (a sub-millisecond window) could leave a
+// stale lock — the error message tells the operator how to clear it.
+function writeProjection({ model, projectionDir, projectRoot, fsImpl } = {}) {
+	const io = fsImpl || fs;
+	const dir = resolveProjectionDir(projectionDir, projectRoot);
+	const { files } = serializeProjection(model);
+
+	io.mkdirSync(dir, { recursive: true });
+
+	// Per-write unique tag so concurrent exports to the same dir never collide on
+	// each other's temp files (which would corrupt snapshots / rollback). pid +
+	// random bytes is unique without depending on wall-clock time.
+	const tempTag = `${process.pid}-${crypto.randomBytes(8).toString('hex')}`;
+	const tempPaths = [];
+	let bytes = 0;
+	for (const name of PROJECTION_FILE_ORDER) {
+		const content = files[name];
+		const tempPath = path.join(dir, `.tmp-${tempTag}-${name}`);
+		io.writeFileSync(tempPath, content);
+		tempPaths.push(tempPath);
+		bytes += Buffer.byteLength(content, 'utf8');
+	}
+
+	const lockPath = path.join(dir, PROJECTION_LOCK_FILE);
+	let lockHeld = false;
+	try {
+		acquireProjectionLock(io, lockPath);
+		lockHeld = true;
+
+		const snapshots = [];
+		const renamed = [];
+		try {
+			for (let i = 0; i < PROJECTION_FILE_ORDER.length; i += 1) {
+				const name = PROJECTION_FILE_ORDER[i];
+				const target = path.join(dir, name);
+				const existed = io.existsSync(target);
+				snapshots.push({ target, existed, content: existed ? io.readFileSync(target) : null });
+				io.renameSync(tempPaths[i], target);
+				renamed.push(target);
+			}
+		} catch (error) {
+			rollbackProjectionWrite(io, snapshots, renamed);
+			throw error;
+		}
+	} finally {
+		// only release a lock we actually acquired (never delete a foreign lock)
+		if (lockHeld) removeIfPresent(io, lockPath);
+		cleanupTempFiles(io, tempPaths);
+	}
+
+	return {
+		dir,
+		writes: PROJECTION_FILE_ORDER.length,
+		bytes,
+		files: [...PROJECTION_FILE_ORDER],
+	};
+}
+
+// Acquire an exclusive publish lock via O_EXCL (flag 'wx'): fails if the lock
+// already exists. Translates the raw EEXIST into an actionable message.
+function acquireProjectionLock(io, lockPath) {
+	try {
+		io.writeFileSync(lockPath, String(process.pid), { flag: 'wx' });
+	} catch (error) {
+		if (error && error.code === 'EEXIST') {
+			throw new Error(
+				`Another forge export is already publishing to this directory (lock: ${lockPath}). `
+				+ 'Retry once it finishes, or remove the lock file if it is stale.',
+			);
+		}
+		throw error;
+	}
+}
+
+function rollbackProjectionWrite(io, snapshots, renamed) {
+	const renamedSet = new Set(renamed);
+	for (const snapshot of snapshots) {
+		if (!renamedSet.has(snapshot.target)) {
+			continue;
+		}
+		if (snapshot.existed) {
+			io.writeFileSync(snapshot.target, snapshot.content);
+		} else {
+			removeIfPresent(io, snapshot.target);
+		}
+	}
+}
+
+function cleanupTempFiles(io, tempPaths) {
+	for (const tempPath of tempPaths) {
+		removeIfPresent(io, tempPath);
+	}
+}
+
+function removeIfPresent(io, target) {
+	try {
+		if (io.existsSync(target)) {
+			io.rmSync(target, { force: true });
+		}
+	} catch { /* intentional: best-effort cleanup, never mask the original failure */ // NOSONAR S2486
+		// no-op
+	}
+}
+
+// Symmetric on-disk read of a projection snapshot. Returns null when no snapshot
+// exists (no manifest), otherwise the imported model with its integrity verified
+// by importProjection (throws on a tampered/corrupt manifest hash).
+function readProjection({ projectionDir, projectRoot, fsImpl } = {}) {
+	const io = fsImpl || fs;
+	const dir = resolveProjectionDir(projectionDir, projectRoot);
+	const manifestPath = path.join(dir, 'manifest.json');
+	if (!io.existsSync(manifestPath)) {
+		return null;
+	}
+
+	const files = {};
+	for (const name of PROJECTION_FILE_ORDER) {
+		const target = path.join(dir, name);
+		files[name] = io.existsSync(target) ? io.readFileSync(target, 'utf8') : '';
+	}
+
+	return { dir, model: importProjection(files) };
+}
+
+const DEFAULT_PROJECTION_TARGET = 'jsonl';
+const DEFAULT_MAX_ATTEMPTS = 5;
+const DEFAULT_BASE_BACKOFF_MS = 5000;
+
+function computeBackoff(now, attempts, baseBackoffMs) {
+	const start = Date.parse(now);
+	if (Number.isNaN(start)) {
+		// Without this guard, new Date(NaN).toISOString() throws a bare
+		// "Invalid time value" RangeError inside the consumer's catch block,
+		// masking the original write error. Fail with an attributable message.
+		throw new RangeError(`computeBackoff: invalid 'now' timestamp: ${now}`);
+	}
+	const delay = baseBackoffMs * 2 ** Math.max(0, attempts - 1);
+	return new Date(start + delay).toISOString();
+}
+
+// Outbox-driven JSONL consumer. Pending kernel_outbox rows for the projection
+// target are dirty markers; draining them performs ONE full-snapshot write that
+// covers all of them. On success every drained row is marked delivered. On write
+// failure each row increments its attempt count: rows below maxAttempts return to
+// pending with an exponential backoff; rows that reach maxAttempts are
+// dead-lettered. A projection failure never mutates Kernel authority.
+async function runJsonlProjectionConsumer({
+	broker,
+	projectionDir,
+	projectRoot,
+	now = new Date().toISOString(),
+	maxAttempts = DEFAULT_MAX_ATTEMPTS,
+	baseBackoffMs = DEFAULT_BASE_BACKOFF_MS,
+	target = DEFAULT_PROJECTION_TARGET,
+	writer = writeProjection,
+} = {}) {
+	const pending = (await broker.listProjectionOutbox({ target, status: 'pending', now })) || [];
+	if (pending.length === 0) {
+		return { drained: 0, written: false, delivered: [], retried: [], dead: [] };
+	}
+
+	const model = await broker.loadProjectionModel();
+
+	let write;
+	try {
+		write = writer({ model, projectionDir, projectRoot });
+	} catch (error) {
+		const retried = [];
+		const dead = [];
+		for (const entry of pending) {
+			const attempts = (entry.attempts || 0) + 1;
+			if (attempts >= maxAttempts) {
+				await broker.deadLetterProjection({
+					outbox_id: entry.id,
+					target,
+					error: error.message,
+					payload_json: JSON.stringify({ event_id: entry.event_id, attempts }),
+					now,
+				});
+				dead.push(entry.id);
+			} else {
+				await broker.recordProjectionFailure({
+					id: entry.id,
+					attempts,
+					next_attempt_at: computeBackoff(now, attempts, baseBackoffMs),
+					error: error.message,
+					now,
+				});
+				retried.push(entry.id);
+			}
+		}
+		return { drained: pending.length, written: false, error: error.message, delivered: [], retried, dead };
+	}
+
+	const delivered = pending.map(entry => entry.id);
+	await broker.markProjectionDelivered(delivered, { now });
+
+	return { drained: pending.length, written: true, write, delivered, retried: [], dead: [] };
+}
+
+module.exports = {
+	SCHEMA_VERSION,
+	DEFAULT_PROJECTION_DIR,
+	DEFAULT_PROJECTION_TARGET,
+	DEFAULT_MAX_ATTEMPTS,
+	DEFAULT_BASE_BACKOFF_MS,
+	PROJECTION_FILE_ORDER,
+	normalizeProjectionModel,
+	buildProjectionSnapshot,
+	serializeProjection,
+	importProjection,
+	resolveProjectionDir,
+	writeProjection,
+	readProjection,
+	computeBackoff,
+	runJsonlProjectionConsumer,
+};