@ghcrawl/api-core 0.1.0

package/src/config.test.ts

@@ -0,0 +1,247 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+
+import {
+  getConfigPath,
+  getTuiRepositoryPreference,
+  isLikelyGitHubToken,
+  isLikelyOpenAiApiKey,
+  loadConfig,
+  readPersistedConfig,
+  writeTuiRepositoryPreference,
+  writePersistedConfig,
+} from './config.js';
+
+function makeTempHome(): string {
+  return fs.mkdtempSync(path.join(os.tmpdir(), 'ghcrawl-config-test-'));
+}
+
+function makeTestEnv(overrides: NodeJS.ProcessEnv = {}): NodeJS.ProcessEnv {
+  return {
+    ...process.env,
+    XDG_CONFIG_HOME: undefined,
+    APPDATA: undefined,
+    ...overrides,
+  };
+}
+
+test('loadConfig prefers persisted config and stores defaults under the user config directory', () => {
+  const home = makeTempHome();
+  const workspace = fs.mkdtempSync(path.join(os.tmpdir(), 'ghcrawl-workspace-'));
+  fs.writeFileSync(path.join(workspace, 'pnpm-workspace.yaml'), 'packages:\n  - "packages/*"\n');
+  const env = {
+    ...makeTestEnv(),
+    HOME: home,
+  };
+
+  writePersistedConfig(
+    {
+      githubToken: 'ghp_testtoken1234567890',
+      openaiApiKey: 'sk-proj-testkey1234567890',
+      apiPort: 6123,
+      embedConcurrency: 12,
+    },
+    { env },
+  );
+
+  const config = loadConfig({ cwd: workspace, env });
+  assert.equal(config.configPath, path.join(home, '.config', 'ghcrawl', 'config.json'));
+  assert.equal(config.configFileExists, true);
+  assert.equal(config.apiPort, 6123);
+  assert.equal(config.embedConcurrency, 12);
+  assert.equal(config.githubTokenSource, 'config');
+  assert.equal(config.openaiApiKeySource, 'config');
+  assert.equal(config.dbPath, path.join(home, '.config', 'ghcrawl', 'ghcrawl.db'));
+});
+
+test('loadConfig lets environment variables override persisted config', () => {
+  const home = makeTempHome();
+  const env = {
+    ...makeTestEnv(),
+    HOME: home,
+    GITHUB_TOKEN: 'ghp_override1234567890',
+    GHCRAWL_API_PORT: '7001',
+  };
+
+  writePersistedConfig(
+    {
+      githubToken: 'ghp_stored1234567890',
+      openaiApiKey: 'sk-proj-stored1234567890',
+      apiPort: 6123,
+    },
+    { env },
+  );
+
+  const config = loadConfig({ cwd: process.cwd(), env });
+  assert.equal(config.githubToken, 'ghp_override1234567890');
+  assert.equal(config.githubTokenSource, 'env');
+  assert.equal(config.apiPort, 7001);
+});
+
+test('loadConfig falls back to repo .env.local when no persisted config exists', () => {
+  const workspace = fs.mkdtempSync(path.join(os.tmpdir(), 'ghcrawl-workspace-'));
+  fs.writeFileSync(path.join(workspace, 'pnpm-workspace.yaml'), 'packages:\n  - "packages/*"\n');
+  fs.writeFileSync(
+    path.join(workspace, '.env.local'),
+    ['GITHUB_TOKEN=ghp_dotenv1234567890', 'OPENAI_API_KEY=sk-proj-dotenv1234567890', 'GHCRAWL_API_PORT=6111'].join('\n'),
+  );
+
+  const config = loadConfig({
+    cwd: workspace,
+    env: {
+      ...makeTestEnv(),
+      HOME: makeTempHome(),
+    },
+  });
+
+  assert.equal(config.githubTokenSource, 'dotenv');
+  assert.equal(config.openaiApiKeySource, 'dotenv');
+  assert.equal(config.apiPort, 6111);
+});
+
+test('loadConfig reuses an existing workspace database when no explicit db path is configured', () => {
+  const workspace = fs.mkdtempSync(path.join(os.tmpdir(), 'ghcrawl-workspace-'));
+  fs.writeFileSync(path.join(workspace, 'pnpm-workspace.yaml'), 'packages:\n  - "packages/*"\n');
+  fs.mkdirSync(path.join(workspace, 'data'), { recursive: true });
+  fs.writeFileSync(path.join(workspace, 'data', 'ghcrawl.db'), '');
+
+  const config = loadConfig({
+    cwd: workspace,
+    env: {
+      ...makeTestEnv(),
+      HOME: makeTempHome(),
+    },
+  });
+
+  assert.equal(config.dbPath, path.join(workspace, 'data', 'ghcrawl.db'));
+});
+
+test('writePersistedConfig creates a readable config file', () => {
+  const home = makeTempHome();
+  const env = {
+    ...makeTestEnv(),
+    HOME: home,
+  };
+
+  const { configPath } = writePersistedConfig(
+    {
+      githubToken: 'ghp_testtoken1234567890',
+      openaiApiKey: 'sk-proj-testkey1234567890',
+    },
+    { env },
+  );
+
+  assert.equal(configPath, getConfigPath({ env }));
+  assert.equal(fs.existsSync(configPath), true);
+
+  const persisted = readPersistedConfig({ env });
+  assert.equal(persisted.data.githubToken, 'ghp_testtoken1234567890');
+  assert.equal(persisted.data.openaiApiKey, 'sk-proj-testkey1234567890');
+});
+
+test('loadConfig restores op metadata and repository tui preferences', () => {
+  const home = makeTempHome();
+  const workspace = fs.mkdtempSync(path.join(os.tmpdir(), 'ghcrawl-workspace-'));
+  fs.writeFileSync(path.join(workspace, 'pnpm-workspace.yaml'), 'packages:\n  - "packages/*"\n');
+  const env = {
+    ...makeTestEnv(),
+    HOME: home,
+  };
+
+  writePersistedConfig(
+    {
+      secretProvider: 'op',
+      opVaultName: 'PwrDrvr LLC',
+      opItemName: 'ghcrawl',
+      tuiPreferences: {
+        'openclaw/openclaw': {
+          minClusterSize: 1,
+          sortMode: 'size',
+        },
+      },
+    },
+    { env },
+  );
+
+  const config = loadConfig({ cwd: workspace, env });
+  assert.equal(config.secretProvider, 'op');
+  assert.equal(config.opVaultName, 'PwrDrvr LLC');
+  assert.equal(config.opItemName, 'ghcrawl');
+  assert.deepEqual(getTuiRepositoryPreference(config, 'openclaw', 'openclaw'), {
+    minClusterSize: 1,
+    sortMode: 'size',
+  });
+});
+
+test('writeTuiRepositoryPreference persists sort and min cluster size by repository', () => {
+  const home = makeTempHome();
+  const workspace = fs.mkdtempSync(path.join(os.tmpdir(), 'ghcrawl-workspace-'));
+  fs.writeFileSync(path.join(workspace, 'pnpm-workspace.yaml'), 'packages:\n  - "packages/*"\n');
+  const env = {
+    ...makeTestEnv(),
+    HOME: home,
+  };
+
+  const config = loadConfig({ cwd: workspace, env });
+  writeTuiRepositoryPreference(config, {
+    owner: 'openclaw',
+    repo: 'openclaw',
+    minClusterSize: 1,
+    sortMode: 'size',
+  });
+
+  const reloaded = loadConfig({ cwd: workspace, env });
+  assert.deepEqual(getTuiRepositoryPreference(reloaded, 'openclaw', 'openclaw'), {
+    minClusterSize: 1,
+    sortMode: 'size',
+  });
+  assert.deepEqual(getTuiRepositoryPreference(reloaded, 'other', 'repo'), {
+    minClusterSize: 10,
+    sortMode: 'recent',
+  });
+});
+
+test('getConfigPath uses APPDATA on Windows', () => {
+  const configPath = getConfigPath({
+    env: {
+      ...makeTestEnv(),
+      APPDATA: 'C:\\Users\\example\\AppData\\Roaming',
+    },
+    platform: 'win32',
+  });
+
+  assert.equal(configPath, path.win32.resolve('C:\\Users\\example\\AppData\\Roaming', 'ghcrawl', 'config.json'));
+});
+
+test('loadConfig rejects invalid port', () => {
+  const home = makeTempHome();
+  assert.throws(() =>
+    loadConfig({
+      cwd: process.cwd(),
+      env: { ...makeTestEnv(), HOME: home, GHCRAWL_API_PORT: 'abc' },
+    }),
+  );
+});
+
+test('loadConfig rejects invalid embed queue settings', () => {
+  const home = makeTempHome();
+  assert.throws(() =>
+    loadConfig({
+      cwd: process.cwd(),
+      env: { ...makeTestEnv(), HOME: home, GHCRAWL_EMBED_CONCURRENCY: '0' },
+    }),
+  );
+});
+
+test('token format helpers match expected API key shapes', () => {
+  assert.equal(isLikelyGitHubToken('ghp_testtoken1234567890'), true);
+  assert.equal(isLikelyGitHubToken('github_pat_1234567890abcdefghijklmnopqrstuvwxyz'), true);
+  assert.equal(isLikelyGitHubToken('not-a-token'), false);
+
+  assert.equal(isLikelyOpenAiApiKey('sk-proj-testkey1234567890'), true);
+  assert.equal(isLikelyOpenAiApiKey('sk-testkey1234567890'), true);
+  assert.equal(isLikelyOpenAiApiKey('openai-key'), false);
+});

package/src/config.ts

@@ -0,0 +1,421 @@
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+
+import dotenv from 'dotenv';
+
+export type ConfigValueSource = 'env' | 'config' | 'dotenv' | 'default' | 'none';
+export type SecretProvider = 'plaintext' | 'op';
+export type TuiSortPreference = 'recent' | 'size';
+export type TuiMinClusterSize = 0 | 1 | 10 | 20 | 50;
+
+export type TuiRepositoryPreference = {
+  minClusterSize: TuiMinClusterSize;
+  sortMode: TuiSortPreference;
+};
+
+export type PersistedGitcrawlConfig = {
+  githubToken?: string;
+  openaiApiKey?: string;
+  secretProvider?: SecretProvider;
+  opVaultName?: string;
+  opItemName?: string;
+  dbPath?: string;
+  apiPort?: number;
+  summaryModel?: string;
+  embedModel?: string;
+  embedBatchSize?: number;
+  embedConcurrency?: number;
+  embedMaxUnread?: number;
+  openSearchUrl?: string;
+  openSearchIndex?: string;
+  tuiPreferences?: Record<string, TuiRepositoryPreference>;
+};
+
+export type GitcrawlConfig = {
+  workspaceRoot: string;
+  configDir: string;
+  configPath: string;
+  configFileExists: boolean;
+  dbPath: string;
+  dbPathSource: ConfigValueSource;
+  apiPort: number;
+  githubToken?: string;
+  githubTokenSource: ConfigValueSource;
+  openaiApiKey?: string;
+  openaiApiKeySource: ConfigValueSource;
+  secretProvider: SecretProvider;
+  opVaultName?: string;
+  opItemName?: string;
+  summaryModel: string;
+  embedModel: string;
+  embedBatchSize: number;
+  embedConcurrency: number;
+  embedMaxUnread: number;
+  openSearchUrl?: string;
+  openSearchIndex: string;
+  tuiPreferences: Record<string, TuiRepositoryPreference>;
+};
+
+type LoadedStoredConfig = {
+  configDir: string;
+  configPath: string;
+  exists: boolean;
+  data: PersistedGitcrawlConfig;
+};
+
+type LoadConfigOptions = {
+  cwd?: string;
+  env?: NodeJS.ProcessEnv;
+  platform?: NodeJS.Platform;
+};
+
+type LayeredValue<T> = {
+  source: ConfigValueSource;
+  value: T | undefined;
+};
+
+function pathModuleForPlatform(platform: NodeJS.Platform) {
+  return platform === 'win32' ? path.win32 : path;
+}
+
+function findWorkspaceRoot(start: string): string {
+  let current = path.resolve(start);
+  while (true) {
+    if (fs.existsSync(path.join(current, 'pnpm-workspace.yaml'))) {
+      return current;
+    }
+    const parent = path.dirname(current);
+    if (parent === current) return path.resolve(start);
+    current = parent;
+  }
+}
+
+function resolveHomeDirectory(env: NodeJS.ProcessEnv): string {
+  const home = env.HOME ?? env.USERPROFILE ?? os.homedir();
+  return path.resolve(home);
+}
+
+export function getConfigDir(options: LoadConfigOptions = {}): string {
+  const env = options.env ?? process.env;
+  const platform = options.platform ?? process.platform;
+  const pathModule = pathModuleForPlatform(platform);
+  if (env.XDG_CONFIG_HOME) {
+    return pathModule.resolve(env.XDG_CONFIG_HOME, 'ghcrawl');
+  }
+  if (platform === 'win32' && env.APPDATA) {
+    return pathModule.resolve(env.APPDATA, 'ghcrawl');
+  }
+  return pathModule.join(resolveHomeDirectory(env), '.config', 'ghcrawl');
+}
+
+export function getConfigPath(options: LoadConfigOptions = {}): string {
+  const platform = options.platform ?? process.platform;
+  const pathModule = pathModuleForPlatform(platform);
+  return pathModule.join(getConfigDir(options), 'config.json');
+}
+
+function readDotenvFile(workspaceRoot: string): Record<string, string> {
+  const dotenvPath = path.join(workspaceRoot, '.env.local');
+  if (!fs.existsSync(dotenvPath)) {
+    return {};
+  }
+  return dotenv.parse(fs.readFileSync(dotenvPath, 'utf8'));
+}
+
+function pickDefined<T>(...values: Array<LayeredValue<T>>): LayeredValue<T> {
+  for (const entry of values) {
+    if (entry.value !== undefined && entry.value !== null) {
+      return entry;
+    }
+  }
+  return { source: 'none', value: undefined };
+}
+
+function getString(value: unknown): string | undefined {
+  return typeof value === 'string' && value.trim().length > 0 ? value : undefined;
+}
+
+function getEnvString(env: NodeJS.ProcessEnv, primary: string, legacy?: string): string | undefined {
+  return getString(env[primary]) ?? (legacy ? getString(env[legacy]) : undefined);
+}
+
+function getDotenvString(values: Record<string, string>, primary: string, legacy?: string): string | undefined {
+  return getString(values[primary]) ?? (legacy ? getString(values[legacy]) : undefined);
+}
+
+function getNumber(value: unknown): number | undefined {
+  return typeof value === 'number' && Number.isFinite(value) ? value : undefined;
+}
+
+function getSecretProvider(value: unknown): SecretProvider | undefined {
+  return value === 'plaintext' || value === 'op' ? value : undefined;
+}
+
+function getTuiSortPreference(value: unknown): TuiSortPreference | undefined {
+  return value === 'recent' || value === 'size' ? value : undefined;
+}
+
+function getTuiMinClusterSize(value: unknown): TuiMinClusterSize | undefined {
+  return value === 0 || value === 1 || value === 10 || value === 20 || value === 50 ? value : undefined;
+}
+
+function getTuiPreferences(value: unknown): Record<string, TuiRepositoryPreference> | undefined {
+  if (!value || typeof value !== 'object') {
+    return undefined;
+  }
+
+  const preferences: Record<string, TuiRepositoryPreference> = {};
+  for (const [fullName, preference] of Object.entries(value as Record<string, unknown>)) {
+    if (!preference || typeof preference !== 'object') {
+      continue;
+    }
+    const record = preference as Record<string, unknown>;
+    const minClusterSize = getTuiMinClusterSize(record.minClusterSize);
+    const sortMode = getTuiSortPreference(record.sortMode);
+    if (minClusterSize === undefined || sortMode === undefined) {
+      continue;
+    }
+    preferences[fullName] = { minClusterSize, sortMode };
+  }
+
+  return preferences;
+}
+
+export function readPersistedConfig(options: LoadConfigOptions = {}): LoadedStoredConfig {
+  const configDir = getConfigDir(options);
+  const configPath = getConfigPath(options);
+  if (!fs.existsSync(configPath)) {
+    return { configDir, configPath, exists: false, data: {} };
+  }
+
+  const raw = JSON.parse(fs.readFileSync(configPath, 'utf8')) as Record<string, unknown>;
+  return {
+    configDir,
+    configPath,
+    exists: true,
+    data: {
+      githubToken: getString(raw.githubToken),
+      openaiApiKey: getString(raw.openaiApiKey),
+      secretProvider: getSecretProvider(raw.secretProvider),
+      opVaultName: getString(raw.opVaultName),
+      opItemName: getString(raw.opItemName),
+      dbPath: getString(raw.dbPath),
+      apiPort: getNumber(raw.apiPort),
+      summaryModel: getString(raw.summaryModel),
+      embedModel: getString(raw.embedModel),
+      embedBatchSize: getNumber(raw.embedBatchSize),
+      embedConcurrency: getNumber(raw.embedConcurrency),
+      embedMaxUnread: getNumber(raw.embedMaxUnread),
+      openSearchUrl: getString(raw.openSearchUrl),
+      openSearchIndex: getString(raw.openSearchIndex),
+      tuiPreferences: getTuiPreferences(raw.tuiPreferences),
+    },
+  };
+}
+
+export function writePersistedConfig(values: PersistedGitcrawlConfig, options: LoadConfigOptions = {}): { configPath: string } {
+  const current = readPersistedConfig(options);
+  fs.mkdirSync(current.configDir, { recursive: true });
+  const next = {
+    ...current.data,
+    ...values,
+  };
+  fs.writeFileSync(current.configPath, `${JSON.stringify(next, null, 2)}\n`, { mode: 0o600 });
+  return { configPath: current.configPath };
+}
+
+function resolveConfiguredPath(configDir: string, value: string): string {
+  return path.isAbsolute(value) ? value : path.resolve(configDir, value);
+}
+
+function getWorkspaceDbPath(workspaceRoot: string): string | null {
+  const workspacePath = path.join(workspaceRoot, 'data', 'ghcrawl.db');
+  return fs.existsSync(workspacePath) ? workspacePath : null;
+}
+
+function parseIntegerSetting(name: string, raw: string): number {
+  const parsed = Number(raw);
+  if (!Number.isSafeInteger(parsed) || parsed <= 0) {
+    throw new Error(`Invalid ${name}: ${raw}`);
+  }
+  return parsed;
+}
+
+export function isLikelyGitHubToken(value: string): boolean {
+  return /^(gh[pousr]_[A-Za-z0-9_]+|github_pat_[A-Za-z0-9_]+)$/.test(value.trim());
+}
+
+export function isLikelyOpenAiApiKey(value: string): boolean {
+  return /^sk-[A-Za-z0-9._-]+$/.test(value.trim());
+}
+
+export function loadConfig(options: LoadConfigOptions = {}): GitcrawlConfig {
+  const cwd = options.cwd ?? process.cwd();
+  const env = options.env ?? process.env;
+  const platform = options.platform ?? process.platform;
+  const workspaceRoot = findWorkspaceRoot(cwd);
+  const stored = readPersistedConfig({ cwd, env, platform });
+  const dotenvValues = readDotenvFile(workspaceRoot);
+
+  const githubToken = pickDefined<string>(
+    { source: 'env', value: getString(env.GITHUB_TOKEN) },
+    { source: 'config', value: stored.data.githubToken },
+    { source: 'dotenv', value: getString(dotenvValues.GITHUB_TOKEN) },
+  );
+  const openaiApiKey = pickDefined<string>(
+    { source: 'env', value: getString(env.OPENAI_API_KEY) },
+    { source: 'config', value: stored.data.openaiApiKey },
+    { source: 'dotenv', value: getString(dotenvValues.OPENAI_API_KEY) },
+  );
+  const configuredDbPath = pickDefined<string>(
+    { source: 'env', value: getEnvString(env, 'GHCRAWL_DB_PATH', 'GHCRAWL_DB_PATH') },
+    { source: 'config', value: stored.data.dbPath },
+    { source: 'dotenv', value: getDotenvString(dotenvValues, 'GHCRAWL_DB_PATH', 'GHCRAWL_DB_PATH') },
+  );
+  const workspaceDbPath = configuredDbPath.value === undefined ? getWorkspaceDbPath(workspaceRoot) : null;
+  const dbPathValue =
+    workspaceDbPath !== null
+      ? { source: 'default' as const, value: workspaceDbPath }
+      : pickDefined<string>(configuredDbPath, { source: 'default', value: 'ghcrawl.db' });
+  const apiPortValue = pickDefined<string | number>(
+    { source: 'env', value: getEnvString(env, 'GHCRAWL_API_PORT', 'GHCRAWL_API_PORT') },
+    { source: 'config', value: stored.data.apiPort },
+    { source: 'dotenv', value: getDotenvString(dotenvValues, 'GHCRAWL_API_PORT', 'GHCRAWL_API_PORT') },
+    { source: 'default', value: '5179' },
+  );
+  const embedBatchSizeValue = pickDefined<string | number>(
+    { source: 'env', value: getEnvString(env, 'GHCRAWL_EMBED_BATCH_SIZE', 'GHCRAWL_EMBED_BATCH_SIZE') },
+    { source: 'config', value: stored.data.embedBatchSize },
+    { source: 'dotenv', value: getDotenvString(dotenvValues, 'GHCRAWL_EMBED_BATCH_SIZE', 'GHCRAWL_EMBED_BATCH_SIZE') },
+    { source: 'default', value: '8' },
+  );
+  const embedConcurrencyValue = pickDefined<string | number>(
+    { source: 'env', value: getEnvString(env, 'GHCRAWL_EMBED_CONCURRENCY', 'GHCRAWL_EMBED_CONCURRENCY') },
+    { source: 'config', value: stored.data.embedConcurrency },
+    { source: 'dotenv', value: getDotenvString(dotenvValues, 'GHCRAWL_EMBED_CONCURRENCY', 'GHCRAWL_EMBED_CONCURRENCY') },
+    { source: 'default', value: '10' },
+  );
+  const embedMaxUnreadValue = pickDefined<string | number>(
+    { source: 'env', value: getEnvString(env, 'GHCRAWL_EMBED_MAX_UNREAD', 'GHCRAWL_EMBED_MAX_UNREAD') },
+    { source: 'config', value: stored.data.embedMaxUnread },
+    { source: 'dotenv', value: getDotenvString(dotenvValues, 'GHCRAWL_EMBED_MAX_UNREAD', 'GHCRAWL_EMBED_MAX_UNREAD') },
+    { source: 'default', value: '20' },
+  );
+  const summaryModel = pickDefined<string>(
+    { source: 'env', value: getEnvString(env, 'GHCRAWL_SUMMARY_MODEL', 'GHCRAWL_SUMMARY_MODEL') },
+    { source: 'config', value: stored.data.summaryModel },
+    { source: 'dotenv', value: getDotenvString(dotenvValues, 'GHCRAWL_SUMMARY_MODEL', 'GHCRAWL_SUMMARY_MODEL') },
+    { source: 'default', value: 'gpt-5-mini' },
+  );
+  const embedModel = pickDefined<string>(
+    { source: 'env', value: getEnvString(env, 'GHCRAWL_EMBED_MODEL', 'GHCRAWL_EMBED_MODEL') },
+    { source: 'config', value: stored.data.embedModel },
+    { source: 'dotenv', value: getDotenvString(dotenvValues, 'GHCRAWL_EMBED_MODEL', 'GHCRAWL_EMBED_MODEL') },
+    { source: 'default', value: 'text-embedding-3-large' },
+  );
+  const openSearchUrl = pickDefined<string>(
+    { source: 'env', value: getEnvString(env, 'GHCRAWL_OPENSEARCH_URL', 'GHCRAWL_OPENSEARCH_URL') },
+    { source: 'config', value: stored.data.openSearchUrl },
+    { source: 'dotenv', value: getDotenvString(dotenvValues, 'GHCRAWL_OPENSEARCH_URL', 'GHCRAWL_OPENSEARCH_URL') },
+  );
+  const openSearchIndex = pickDefined<string>(
+    { source: 'env', value: getEnvString(env, 'GHCRAWL_OPENSEARCH_INDEX', 'GHCRAWL_OPENSEARCH_INDEX') },
+    { source: 'config', value: stored.data.openSearchIndex },
+    { source: 'dotenv', value: getDotenvString(dotenvValues, 'GHCRAWL_OPENSEARCH_INDEX', 'GHCRAWL_OPENSEARCH_INDEX') },
+    { source: 'default', value: 'ghcrawl-threads' },
+  );
+
+  const dbPath =
+    dbPathValue.value && path.isAbsolute(dbPathValue.value)
+      ? dbPathValue.value
+      : resolveConfiguredPath(stored.configDir, dbPathValue.value ?? 'ghcrawl.db');
+  const apiPort = parseIntegerSetting('GHCRAWL_API_PORT', String(apiPortValue.value ?? '5179'));
+  const embedBatchSize = parseIntegerSetting('GHCRAWL_EMBED_BATCH_SIZE', String(embedBatchSizeValue.value ?? '8'));
+  const embedConcurrency = parseIntegerSetting('GHCRAWL_EMBED_CONCURRENCY', String(embedConcurrencyValue.value ?? '10'));
+  const embedMaxUnread = parseIntegerSetting('GHCRAWL_EMBED_MAX_UNREAD', String(embedMaxUnreadValue.value ?? '20'));
+
+  return {
+    workspaceRoot,
+    configDir: stored.configDir,
+    configPath: stored.configPath,
+    configFileExists: stored.exists,
+    dbPath,
+    dbPathSource: dbPathValue.source,
+    apiPort,
+    githubToken: githubToken.value,
+    githubTokenSource: githubToken.source,
+    openaiApiKey: openaiApiKey.value,
+    openaiApiKeySource: openaiApiKey.source,
+    secretProvider: stored.data.secretProvider ?? 'plaintext',
+    opVaultName: stored.data.opVaultName,
+    opItemName: stored.data.opItemName,
+    summaryModel: summaryModel.value ?? 'gpt-5-mini',
+    embedModel: embedModel.value ?? 'text-embedding-3-large',
+    embedBatchSize,
+    embedConcurrency,
+    embedMaxUnread,
+    openSearchUrl: openSearchUrl.value,
+    openSearchIndex: openSearchIndex.value ?? 'ghcrawl-threads',
+    tuiPreferences: stored.data.tuiPreferences ?? {},
+  };
+}
+
+export function ensureRuntimeDirs(config: GitcrawlConfig): void {
+  fs.mkdirSync(config.configDir, { recursive: true });
+  fs.mkdirSync(path.dirname(config.dbPath), { recursive: true });
+}
+
+export function getTuiRepositoryPreference(config: GitcrawlConfig, owner: string, repo: string): TuiRepositoryPreference {
+  return config.tuiPreferences[`${owner}/${repo}`] ?? { minClusterSize: 10, sortMode: 'recent' };
+}
+
+export function writeTuiRepositoryPreference(
+  config: GitcrawlConfig,
+  params: { owner: string; repo: string; minClusterSize: TuiMinClusterSize; sortMode: TuiSortPreference },
+): { configPath: string } {
+  const fullName = `${params.owner}/${params.repo}`;
+  const nextPreferences = {
+    ...config.tuiPreferences,
+    [fullName]: {
+      minClusterSize: params.minClusterSize,
+      sortMode: params.sortMode,
+    },
+  };
+  config.tuiPreferences = nextPreferences;
+  const next = fs.existsSync(config.configPath)
+    ? ({
+        ...(JSON.parse(fs.readFileSync(config.configPath, 'utf8')) as PersistedGitcrawlConfig),
+        tuiPreferences: nextPreferences,
+      } satisfies PersistedGitcrawlConfig)
+    : ({
+        tuiPreferences: nextPreferences,
+      } satisfies PersistedGitcrawlConfig);
+  fs.mkdirSync(config.configDir, { recursive: true });
+  fs.writeFileSync(config.configPath, `${JSON.stringify(next, null, 2)}\n`, { mode: 0o600 });
+  return { configPath: config.configPath };
+}
+
+export function requireGithubToken(config: GitcrawlConfig): string {
+  if (!config.githubToken) {
+    if (config.secretProvider === 'op' && config.opVaultName && config.opItemName) {
+      throw new Error(
+        `Missing GitHub token in the environment. This config is set to use 1Password CLI via ${config.opVaultName}/${config.opItemName}; run ghcrawl through your op wrapper or set GITHUB_TOKEN. Expected config at ${config.configPath}`,
+      );
+    }
+    throw new Error(`Missing GitHub token. Run ghcrawl init or set GITHUB_TOKEN. Expected config at ${config.configPath}`);
+  }
+  return config.githubToken;
+}
+
+export function requireOpenAiKey(config: GitcrawlConfig): string {
+  if (!config.openaiApiKey) {
+    if (config.secretProvider === 'op' && config.opVaultName && config.opItemName) {
+      throw new Error(
+        `Missing OpenAI API key in the environment. This config is set to use 1Password CLI via ${config.opVaultName}/${config.opItemName}; run ghcrawl through your op wrapper or set OPENAI_API_KEY. Expected config at ${config.configPath}`,
+      );
+    }
+    throw new Error(`Missing OpenAI API key. Run ghcrawl init or set OPENAI_API_KEY. Expected config at ${config.configPath}`);
+  }
+  return config.openaiApiKey;
+}

package/src/db/migrate.test.ts

@@ -0,0 +1,30 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+
+import { migrate } from './migrate.js';
+import { openDb } from './sqlite.js';
+
+test('migrate creates core tables', () => {
+  const db = openDb(':memory:');
+  try {
+    migrate(db);
+    const rows = db
+      .prepare("select name from sqlite_master where type in ('table', 'view') order by name asc")
+      .all() as Array<{ name: string }>;
+    const names = rows.map((row) => row.name);
+
+    assert.ok(names.includes('repositories'));
+    assert.ok(names.includes('threads'));
+    assert.ok(names.includes('documents'));
+    assert.ok(names.includes('document_embeddings'));
+    assert.ok(names.includes('cluster_runs'));
+    assert.ok(names.includes('repo_sync_state'));
+
+    const threadColumns = db.prepare('pragma table_info(threads)').all() as Array<{ name: string }>;
+    const threadColumnNames = threadColumns.map((column) => column.name);
+    assert.ok(threadColumnNames.includes('first_pulled_at'));
+    assert.ok(threadColumnNames.includes('last_pulled_at'));
+  } finally {
+    db.close();
+  }
+});