langflower 0.0.10 → 0.1.0

package/vendor/server/skeleton/nodes/hello-embed/lib/ingest-search.test.ts

@@ -0,0 +1,180 @@
+import fs from 'node:fs/promises';
+import os from 'node:os';
+import path from 'node:path';
+import { afterEach, beforeEach, describe, expect, it } from 'vitest';
+import { runIngest } from './ingest.ts';
+import { DEFAULT_SQLITE_PATH } from './paths.ts';
+import { runSearch } from './search.ts';
+import type { TextEmbedder } from './text-embedder.ts';
+import { l2Normalize } from './vectors.ts';
+
+const fakeEmbedder = (): TextEmbedder => ({
+	expectedDim: 8,
+	embedTexts: async (texts) =>
+		texts.map((text) => {
+			const values = new Array<number>(8).fill(0);
+			if (text.toLowerCase().includes('alpha')) {
+				values[0] = 1;
+			} else if (text.toLowerCase().includes('beta')) {
+				values[1] = 1;
+			} else {
+				values[7] = 1;
+			}
+			return l2Normalize(values);
+		}),
+});
+
+const collect = async <T>(iter: AsyncIterable<T>): Promise<readonly T[]> => {
+	const events: T[] = [];
+	for await (const event of iter) {
+		events.push(event);
+	}
+	return events;
+};
+
+const progressTexts = (
+	events: readonly { readonly kind: string; readonly text?: string }[],
+): readonly string[] =>
+	events.flatMap((event) =>
+		event.kind === 'progress' && event.text !== undefined
+			? [event.text]
+			: [],
+	);
+
+describe('ingest + search libs', () => {
+	let root: string;
+
+	beforeEach(async () => {
+		root = await fs.mkdtemp(path.join(os.tmpdir(), 'hello-embed-lib-'));
+		await fs.writeFile(
+			path.join(root, 'notes.md'),
+			'# Alpha\nalpha body\n\n# Beta\nbeta body\n',
+			'utf8',
+		);
+	});
+
+	afterEach(async () => {
+		await fs.rm(root, { recursive: true, force: true });
+	});
+
+	it('indexes heading chunks and ranks the matching query first', async () => {
+		const sqlitePath = path.join(root, DEFAULT_SQLITE_PATH);
+		const events = await collect(
+			runIngest({
+				sqlitePath,
+				sourceDir: root,
+				embedder: fakeEmbedder(),
+			}),
+		);
+		expect(events.at(-1)).toMatchObject({
+			kind: 'finish',
+			chunkCount: 2,
+		});
+
+		const result = await runSearch({
+			sqlitePath,
+			query: 'alpha',
+			topK: 8,
+			embedder: fakeEmbedder(),
+		});
+		expect(result.hits[0]?.heading).toBe('Alpha');
+		expect(result.text).toContain('Question:\n');
+		expect(result.text).toContain('alpha');
+		expect(result.text).toContain('Context:');
+		expect(result.text).toContain('notes.md');
+	});
+
+	it('prefixes progress with a zero-padded chunk counter', async () => {
+		const events = await collect(
+			runIngest({
+				sqlitePath: path.join(root, DEFAULT_SQLITE_PATH),
+				sourceDir: root,
+				embedder: fakeEmbedder(),
+			}),
+		);
+		expect(progressTexts(events)).toEqual([
+			'(1/2) notes.md — Alpha',
+			'(2/2) notes.md — Beta',
+		]);
+	});
+
+	it('pads the progress counter to the digit width of the total', async () => {
+		const headings = Array.from(
+			{ length: 10 },
+			(_, index) => `# H${String(index)}\nbody\n`,
+		).join('\n');
+		await fs.writeFile(path.join(root, 'notes.md'), headings, 'utf8');
+		const events = await collect(
+			runIngest({
+				sqlitePath: path.join(root, DEFAULT_SQLITE_PATH),
+				sourceDir: root,
+				embedder: fakeEmbedder(),
+			}),
+		);
+		const lines = progressTexts(events);
+		expect(lines[0]).toBe('(01/10) notes.md — H0');
+		expect(lines.at(-1)).toBe('(10/10) notes.md — H9');
+		expect(lines).toHaveLength(10);
+	});
+
+	it('errors when the sqlite file is missing', async () => {
+		await expect(
+			runSearch({
+				sqlitePath: path.join(root, 'missing.sqlite'),
+				query: 'alpha',
+				topK: 8,
+				embedder: fakeEmbedder(),
+			}),
+		).rejects.toThrow(/Run kb-ingest first/);
+	});
+
+	it('packs full chunk bodies into Question/Context text', async () => {
+		const longBody = `alpha ${'x'.repeat(280)}`;
+		await fs.writeFile(
+			path.join(root, 'notes.md'),
+			`# Alpha\n${longBody}\n`,
+			'utf8',
+		);
+		const sqlitePath = path.join(root, DEFAULT_SQLITE_PATH);
+		await collect(
+			runIngest({
+				sqlitePath,
+				sourceDir: root,
+				embedder: fakeEmbedder(),
+			}),
+		);
+		const result = await runSearch({
+			sqlitePath,
+			query: 'alpha',
+			topK: 8,
+			embedder: fakeEmbedder(),
+		});
+		expect(result.text).toContain('Question:\nalpha');
+		expect(result.text).toContain(longBody);
+		expect(result.text.includes(`${'x'.repeat(237)}…`)).toBe(false);
+	});
+
+	it('ranks a keyword-only FTS hit above equal cosine dummies', async () => {
+		await fs.writeFile(
+			path.join(root, 'notes.md'),
+			'# Alpha\nalpha body\n\n# Beta\nbeta body xylophoneunique\n',
+			'utf8',
+		);
+		const sqlitePath = path.join(root, DEFAULT_SQLITE_PATH);
+		await collect(
+			runIngest({
+				sqlitePath,
+				sourceDir: root,
+				embedder: fakeEmbedder(),
+			}),
+		);
+		const result = await runSearch({
+			sqlitePath,
+			query: 'xylophoneunique',
+			topK: 8,
+			embedder: fakeEmbedder(),
+		});
+		expect(result.hits[0]?.heading).toBe('Beta');
+		expect(result.text).toContain('xylophoneunique');
+	});
+});

package/vendor/server/skeleton/nodes/hello-embed/lib/ingest.ts

@@ -0,0 +1,128 @@
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import { DatabaseSync } from 'node:sqlite';
+import { chunkMarkdown } from './chunk-markdown.ts';
+import { SCHEMA_SQL } from './schema.ts';
+import type { TextEmbedder } from './text-embedder.ts';
+import { walkMarkdown } from './walk-markdown.ts';
+import { clearVecCache, float32ToBlob, l2Normalize } from './vectors.ts';
+
+export type IngestProgressEvent = {
+	readonly kind: 'progress';
+	readonly text: string;
+};
+
+export type IngestFinishEvent = {
+	readonly kind: 'finish';
+	readonly chunkCount: number;
+	readonly fileCount: number;
+};
+
+export type IngestEvent = IngestProgressEvent | IngestFinishEvent;
+
+export type RunIngestOptions = {
+	readonly sqlitePath: string;
+	readonly sourceDir: string;
+	readonly embedder: TextEmbedder;
+};
+
+const dimMismatchMessage = (got: number, stored: number): string =>
+	`Embedding dim ${String(got)} does not match stored dim ${String(stored)}. Re-run ingest after switching models.`;
+
+const padCount = (value: number, width: number): string =>
+	String(value).padStart(width, '0');
+
+export const runIngest = async function* (
+	options: RunIngestOptions,
+): AsyncGenerator<IngestEvent> {
+	const { sqlitePath, sourceDir, embedder } = options;
+	await fs.mkdir(path.dirname(sqlitePath), { recursive: true });
+	try {
+		await fs.unlink(sqlitePath);
+	} catch (error) {
+		const code =
+			error !== null && typeof error === 'object' && 'code' in error
+				? String((error as { code: unknown }).code)
+				: '';
+		if (code !== 'ENOENT') {
+			throw error;
+		}
+	}
+
+	const db = new DatabaseSync(sqlitePath);
+	try {
+		db.exec('PRAGMA journal_mode = WAL');
+		db.exec('PRAGMA foreign_keys = ON');
+		db.exec(SCHEMA_SQL);
+		clearVecCache();
+
+		const files = await walkMarkdown(sourceDir);
+		const readable = files.filter((file) => file.skipped === undefined);
+		for (const file of files) {
+			if (file.skipped !== undefined) {
+				yield { kind: 'progress', text: file.skipped };
+			}
+		}
+
+		if (readable.length === 0) {
+			yield { kind: 'progress', text: 'no markdown files' };
+			yield { kind: 'finish', chunkCount: 0, fileCount: 0 };
+			return;
+		}
+
+		const chunks = readable.flatMap((file) =>
+			chunkMarkdown(file.relPath, file.text),
+		);
+		const insertChunk = db.prepare(
+			'INSERT INTO chunk (id, path, heading, text, embed_text) VALUES (?, ?, ?, ?, ?)',
+		);
+		const insertVec = db.prepare(
+			'INSERT INTO chunk_vec (id, dim, vector) VALUES (?, ?, ?)',
+		);
+		const insertFts = db.prepare(
+			'INSERT INTO chunk_fts (id, heading, text) VALUES (?, ?, ?)',
+		);
+
+		let expectedDim = embedder.expectedDim;
+		let embedded = 0;
+		const width = String(chunks.length).length;
+		for (const chunk of chunks) {
+			if (chunk.truncated !== undefined) {
+				yield { kind: 'progress', text: chunk.truncated };
+			}
+			const [raw] = await embedder.embedTexts([chunk.embedText]);
+			if (raw === undefined) {
+				throw new Error('Embedding returned no vector');
+			}
+			const vector = l2Normalize([...raw]);
+			if (expectedDim === undefined) {
+				expectedDim = vector.length;
+			} else if (vector.length !== expectedDim) {
+				throw new Error(dimMismatchMessage(vector.length, expectedDim));
+			}
+			insertChunk.run(
+				chunk.id,
+				chunk.path,
+				chunk.heading,
+				chunk.text,
+				chunk.embedText,
+			);
+			insertVec.run(chunk.id, expectedDim, float32ToBlob(vector));
+			insertFts.run(chunk.id, chunk.heading, chunk.text);
+			embedded += 1;
+			const label = chunk.heading.length > 0 ? chunk.heading : '(intro)';
+			yield {
+				kind: 'progress',
+				text: `(${padCount(embedded, width)}/${padCount(chunks.length, width)}) ${chunk.path} — ${label}`,
+			};
+		}
+
+		yield {
+			kind: 'finish',
+			chunkCount: embedded,
+			fileCount: readable.length,
+		};
+	} finally {
+		db.close();
+	}
+};

package/vendor/server/skeleton/nodes/hello-embed/lib/paths.test.ts

@@ -0,0 +1,20 @@
+import path from 'node:path';
+import { describe, expect, it } from 'vitest';
+import {
+	DEFAULT_SQLITE_PATH,
+	resolveSqlitePath,
+	resolveUnderProject,
+} from './paths.ts';
+
+describe('resolveUnderProject', () => {
+	it('rejects paths that escape the project', () => {
+		expect(() => resolveUnderProject('/proj', '../outside')).toThrow(
+			/escapes projectDir/,
+		);
+	});
+
+	it('resolves sqlite default under the project', () => {
+		const resolved = resolveSqlitePath('/proj', '');
+		expect(resolved).toBe(path.resolve('/proj', DEFAULT_SQLITE_PATH));
+	});
+});

package/vendor/server/skeleton/nodes/hello-embed/lib/paths.ts

@@ -0,0 +1,70 @@
+import path from 'node:path';
+
+/** Project-relative index path. Same default on every pack node. */
+export const DEFAULT_SQLITE_PATH = '.langflower/.cache/hello-embed/kb.sqlite';
+
+export const DEFAULT_SEARCH_TOP_K = 8;
+
+export const MAX_SEARCH_TOP_K = 50;
+
+export const MAX_FILE_BYTES = 512 * 1024;
+
+export const MAX_CHUNKS_PER_FILE = 200;
+
+export const SKIP_DIR_NAMES = new Set(['node_modules', '.git']);
+
+export const CACHE_PREFIX = '.langflower/.cache';
+
+const toPosix = (value: string): string => value.replaceAll('\\', '/');
+
+export const asPosixRelative = (value: string): string => toPosix(value);
+
+/**
+ * Resolve `relativeOrEmpty` under `projectDir`. Empty means `projectDir`.
+ * Rejects paths that escape the project root.
+ */
+export const resolveUnderProject = (
+	projectDir: string,
+	relativeOrEmpty: string,
+): string => {
+	const root = path.resolve(projectDir);
+	const trimmed = relativeOrEmpty.trim();
+	const resolved =
+		trimmed.length === 0
+			? root
+			: path.isAbsolute(trimmed)
+				? path.resolve(trimmed)
+				: path.resolve(root, trimmed);
+	const rel = path.relative(root, resolved);
+	if (rel.startsWith('..') || path.isAbsolute(rel)) {
+		throw new Error(`Path escapes projectDir: ${trimmed}`);
+	}
+	return resolved;
+};
+
+export const resolveSqlitePath = (
+	projectDir: string,
+	override?: string,
+): string => {
+	const raw =
+		typeof override === 'string' && override.trim().length > 0
+			? override.trim()
+			: DEFAULT_SQLITE_PATH;
+	return resolveUnderProject(projectDir, raw);
+};
+
+export const clampTopK = (value: unknown, fallback: number): number => {
+	const parsed =
+		typeof value === 'number'
+			? value
+			: typeof value === 'string' && value.trim().length > 0
+				? Number(value)
+				: fallback;
+	if (!Number.isFinite(parsed)) {
+		return fallback;
+	}
+	return Math.min(MAX_SEARCH_TOP_K, Math.max(1, Math.floor(parsed)));
+};
+
+export const isCacheRelative = (relPosix: string): boolean =>
+	relPosix === CACHE_PREFIX || relPosix.startsWith(`${CACHE_PREFIX}/`);

package/vendor/server/skeleton/nodes/hello-embed/lib/schema.ts

@@ -0,0 +1,21 @@
+export const SCHEMA_SQL = `
+CREATE TABLE chunk (
+	id TEXT PRIMARY KEY,
+	path TEXT NOT NULL,
+	heading TEXT NOT NULL,
+	text TEXT NOT NULL,
+	embed_text TEXT NOT NULL
+);
+
+CREATE TABLE chunk_vec (
+	id TEXT PRIMARY KEY REFERENCES chunk(id),
+	dim INTEGER NOT NULL,
+	vector BLOB NOT NULL
+);
+
+CREATE VIRTUAL TABLE chunk_fts USING fts5(
+	id UNINDEXED,
+	heading,
+	text
+);
+`.trim();

package/vendor/server/skeleton/nodes/hello-embed/lib/search.ts

@@ -0,0 +1,254 @@
+import fs from 'node:fs';
+import { DatabaseSync } from 'node:sqlite';
+import type { TextEmbedder } from './text-embedder.ts';
+import { l2Normalize, loadVecIndex, topCosine } from './vectors.ts';
+
+export type SearchHit = {
+	readonly path: string;
+	readonly heading: string;
+	readonly score: number;
+	readonly text: string;
+};
+
+export type RunSearchOptions = {
+	readonly sqlitePath: string;
+	readonly query: string;
+	readonly topK: number;
+	readonly embedder: TextEmbedder;
+};
+
+type ChunkRow = {
+	readonly id: string;
+	readonly path: string;
+	readonly heading: string;
+	readonly text: string;
+};
+
+type RankedId = {
+	readonly id: string;
+};
+
+const RRF_K = 60;
+
+const FTS_TOKEN = /[\p{L}\p{N}_]+/gu;
+
+const asChunkRow = (row: unknown): ChunkRow => {
+	if (
+		row === null ||
+		typeof row !== 'object' ||
+		!('id' in row) ||
+		!('path' in row) ||
+		!('heading' in row) ||
+		!('text' in row)
+	) {
+		throw new Error('Unexpected chunk row');
+	}
+	const id = (row as { id: unknown }).id;
+	const path = (row as { path: unknown }).path;
+	const heading = (row as { heading: unknown }).heading;
+	const text = (row as { text: unknown }).text;
+	if (
+		typeof id !== 'string' ||
+		typeof path !== 'string' ||
+		typeof heading !== 'string' ||
+		typeof text !== 'string'
+	) {
+		throw new Error('chunk row has invalid types');
+	}
+	return { id, path, heading, text };
+};
+
+type VecRow = {
+	readonly id: string;
+	readonly dim: number;
+	readonly vector: Uint8Array;
+};
+
+const asVecRow = (row: unknown): VecRow => {
+	if (
+		row === null ||
+		typeof row !== 'object' ||
+		!('id' in row) ||
+		!('dim' in row) ||
+		!('vector' in row)
+	) {
+		throw new Error('Unexpected chunk_vec row');
+	}
+	const id = (row as { id: unknown }).id;
+	const dimRaw = (row as { dim: unknown }).dim;
+	const vector = (row as { vector: unknown }).vector;
+	const dim = typeof dimRaw === 'bigint' ? Number(dimRaw) : dimRaw;
+	if (
+		typeof id !== 'string' ||
+		typeof dim !== 'number' ||
+		!(vector instanceof Uint8Array)
+	) {
+		throw new Error('chunk_vec row has invalid types');
+	}
+	return { id, dim, vector };
+};
+
+const retrieveCandidateLimit = (topK: number): number => Math.max(topK * 4, 32);
+
+/** Quoted FTS5 tokens so chat text cannot inject MATCH operators. */
+export const ftsMatchQuery = (raw: string): string | undefined => {
+	const tokens = raw.match(FTS_TOKEN);
+	if (tokens === null || tokens.length === 0) {
+		return undefined;
+	}
+	return tokens.map((token) => `"${token.replaceAll('"', '')}"`).join(' ');
+};
+
+export const reciprocalRankFusion = (
+	lists: readonly (readonly RankedId[])[],
+	k: number,
+): readonly { readonly id: string; readonly score: number }[] => {
+	const scores = new Map<string, number>();
+	for (const list of lists) {
+		list.forEach((item, index) => {
+			const rank = index + 1;
+			scores.set(item.id, (scores.get(item.id) ?? 0) + 1 / (k + rank));
+		});
+	}
+	return [...scores.entries()]
+		.map(([id, score]) => ({ id, score }))
+		.sort((left, right) => right.score - left.score);
+};
+
+export const formatHitsText = (
+	query: string,
+	hits: readonly SearchHit[],
+): string => {
+	if (hits.length === 0) {
+		return `Question:\n${query}\n\nNo hits.`;
+	}
+	const blocks = hits.map((hit, index) => {
+		const heading = hit.heading.length > 0 ? hit.heading : '(intro)';
+		const score = hit.score.toFixed(2);
+		return `${String(index + 1)}. ${hit.path} — ${heading} (${score})\n${hit.text}`;
+	});
+	return `Question:\n${query}\n\nContext:\n${blocks.join('\n\n')}`;
+};
+
+const hasFtsTable = (db: DatabaseSync): boolean => {
+	const row = db
+		.prepare(
+			`SELECT 1 AS ok FROM sqlite_master WHERE type = 'table' AND name = 'chunk_fts'`,
+		)
+		.get() as { ok?: unknown } | undefined;
+	return row !== undefined;
+};
+
+const ftsRankedIds = (
+	db: DatabaseSync,
+	query: string,
+	limit: number,
+): readonly RankedId[] => {
+	if (!hasFtsTable(db)) {
+		return [];
+	}
+	const match = ftsMatchQuery(query);
+	if (match === undefined) {
+		return [];
+	}
+	try {
+		const rows = db
+			.prepare('SELECT id FROM chunk_fts WHERE chunk_fts MATCH ? LIMIT ?')
+			.all(match, limit);
+		return rows.flatMap((row) => {
+			if (
+				row === null ||
+				typeof row !== 'object' ||
+				!('id' in row) ||
+				typeof (row as { id: unknown }).id !== 'string'
+			) {
+				return [];
+			}
+			return [{ id: (row as { id: string }).id }];
+		});
+	} catch {
+		return [];
+	}
+};
+
+export const runSearch = async (
+	options: RunSearchOptions,
+): Promise<{
+	readonly hits: readonly SearchHit[];
+	readonly text: string;
+}> => {
+	const query = options.query.trim();
+	if (!fs.existsSync(options.sqlitePath)) {
+		throw new Error(
+			`SQLite DB not found: ${options.sqlitePath}. Run kb-ingest first.`,
+		);
+	}
+	if (query.length === 0) {
+		return { hits: [], text: formatHitsText(query, []) };
+	}
+
+	const db = new DatabaseSync(options.sqlitePath);
+	try {
+		const candidateLimit = retrieveCandidateLimit(options.topK);
+		const vecRows = db
+			.prepare('SELECT id, dim, vector FROM chunk_vec')
+			.all()
+			.map((row) => asVecRow(row));
+		const mtimeMs = fs.statSync(options.sqlitePath).mtimeMs;
+		const index = loadVecIndex(
+			options.sqlitePath,
+			vecRows.map((row) => ({
+				id: row.id,
+				dim: row.dim,
+				vector: row.vector,
+			})),
+			mtimeMs,
+		);
+		let cosineList: readonly RankedId[] = [];
+		if (index.dim > 0) {
+			const [raw] = await options.embedder.embedTexts([query]);
+			if (raw === undefined) {
+				throw new Error('Embedding returned no vector');
+			}
+			if (raw.length !== index.dim) {
+				throw new Error(
+					`Embedding dim ${String(raw.length)} does not match stored dim ${String(index.dim)}. Re-run ingest after switching models.`,
+				);
+			}
+			const queryVec = l2Normalize([...raw]);
+			cosineList = topCosine(index, queryVec, candidateLimit).map(
+				(item) => ({ id: item.id }),
+			);
+		}
+		const ftsList = ftsRankedIds(db, query, candidateLimit);
+		const fused = reciprocalRankFusion(
+			[cosineList, ftsList].filter((list) => list.length > 0),
+			RRF_K,
+		).slice(0, options.topK);
+		const byId = new Map(
+			db
+				.prepare('SELECT id, path, heading, text FROM chunk')
+				.all()
+				.map((row) => {
+					const parsed = asChunkRow(row);
+					return [parsed.id, parsed] as const;
+				}),
+		);
+		const hits: SearchHit[] = [];
+		for (const item of fused) {
+			const chunk = byId.get(item.id);
+			if (chunk === undefined) {
+				continue;
+			}
+			hits.push({
+				path: chunk.path,
+				heading: chunk.heading,
+				score: item.score,
+				text: chunk.text,
+			});
+		}
+		return { hits, text: formatHitsText(query, hits) };
+	} finally {
+		db.close();
+	}
+};

package/vendor/server/skeleton/nodes/hello-embed/lib/text-embedder.ts

@@ -0,0 +1,14 @@
+import type { EmbedHandle, EmbedTextRole } from '@langflower/node-sdk';
+
+export type TextEmbedder = {
+	readonly expectedDim?: number;
+	embedTexts: (texts: readonly string[]) => Promise<readonly Float32Array[]>;
+};
+
+export const textEmbedderFromHandle = (
+	handle: EmbedHandle,
+	role: EmbedTextRole,
+): TextEmbedder => ({
+	expectedDim: handle.dim,
+	embedTexts: (texts) => handle.embedTexts(texts, { role }),
+});