@kitkat-vn/kitcode 0.2.0-staging.3

package/src/scan-worker.mjs

@@ -0,0 +1,23 @@
+// The tracker's scanning thread (scanner.mjs starts it). Listing a project, reading and hashing its
+// files, and asking git for its state all block, so they happen here, and the tracker's own thread
+// keeps answering the local API meanwhile. It only reads: every result goes back to the tracker,
+// which folds it into state.json under the state lock. A Git project's read is checked against git
+// here too, since the git position its snapshot recorded (readProjectScan): waiting for a git command
+// to finish writing the index blocks this thread, not the tracker's.
+import {parentPort} from 'node:worker_threads';
+import {countCommits, getGitSignature} from './git.mjs';
+import {readProjectScan, shouldIgnoreRelativePath} from './source-snapshot.mjs';
+
+parentPort.on('message', ({id, kind, root, sourceType, changedPaths, since}) => {
+  try {
+    // The status of files the scanner leaves out (a lockfile, say) is not part of the signature, so
+    // changing one is not activity.
+    const result = kind === 'git'
+      ? {signature: getGitSignature(root, shouldIgnoreRelativePath), commitCount: countCommits(root)}
+      : readProjectScan(root, {sourceType, changedPaths, since});
+
+    parentPort.postMessage({id, ok: true, result});
+  } catch (error) {
+    parentPort.postMessage({id, ok: false, error: {message: error.message, code: error.code}});
+  }
+});

package/src/scanner.mjs

@@ -0,0 +1,84 @@
+import {Worker} from 'node:worker_threads';
+
+// Runs the tracker's scans in a worker thread (scan-worker.mjs), one at a time, in the order asked.
+// A worker that fails is replaced at the next request; `onError` hears about the failure.
+export function createScanner({onError} = {}) {
+  let worker = null;
+  let nextId = 1;
+  const pending = new Map();
+
+  function failPending(error) {
+    for (const {reject} of pending.values()) {
+      reject(error);
+    }
+
+    pending.clear();
+  }
+
+  function start() {
+    const thread = new Worker(new URL('./scan-worker.mjs', import.meta.url));
+    // The tracker's server keeps the process alive; a scan in progress must not.
+    thread.unref();
+
+    thread.on('message', ({id, ok, result, error}) => {
+      const request = pending.get(id);
+
+      if (!request) {
+        return;
+      }
+
+      pending.delete(id);
+
+      if (ok) {
+        request.resolve(result);
+      } else {
+        request.reject(Object.assign(new Error(error.message), {code: error.code}));
+      }
+    });
+
+    const stopped = (error) => {
+      if (worker !== thread) {
+        return;
+      }
+
+      worker = null;
+      onError?.(error);
+      failPending(error);
+    };
+
+    thread.on('error', stopped);
+    thread.on('exit', (code) => stopped(new Error(`the scan worker stopped with exit code ${code}`)));
+    worker = thread;
+  }
+
+  function request(message) {
+    if (!worker) {
+      start();
+    }
+
+    const id = nextId;
+    nextId += 1;
+
+    return new Promise((resolve, reject) => {
+      pending.set(id, {resolve, reject});
+      worker.postMessage({...message, id});
+    });
+  }
+
+  return {
+    // `since` is the git position the project's snapshot recorded: a Git project's read is checked
+    // against it (source-snapshot.mjs readProjectScan).
+    read({root, sourceType, changedPaths = null, since}) {
+      return request({kind: 'read', root, sourceType, changedPaths, since});
+    },
+    git({root}) {
+      return request({kind: 'git', root});
+    },
+    async close() {
+      const thread = worker;
+      worker = null;
+      failPending(new Error('the scanner closed'));
+      await thread?.terminate();
+    },
+  };
+}

package/src/seen-lines.mjs

@@ -0,0 +1,148 @@
+import fs from 'node:fs';
+import {leftoverTempFiles, writeFileAtomically} from './atomic-file.mjs';
+
+// The format of ~/.kitcode/seen-lines, every line KitCode has seen in any of the user's projects
+// (docs/security.md, "The seen-lines file"). A line is the first 8 bytes of its sha256: its key in a
+// snapshot, 16 hex characters, as bytes. The file is a 16-byte header, "KCSEEN01" and the number of
+// lines as an unsigned 64-bit big-endian integer, then the lines, 8 bytes each, in ascending order and
+// each once, so that a line is found by binary search. The store (store.mjs) reads and writes it,
+// under the state lock.
+const MAGIC = Buffer.from('KCSEEN01', 'latin1');
+const HEADER_BYTES = 16;
+const LINE_BYTES = 8;
+const LINE_KEY = /^[0-9a-f]{16}$/;
+const FILE_MODE = 0o600;
+
+export const NO_SEEN_LINES = Buffer.alloc(0);
+
+// Bytes that are not a seen-lines file.
+export class SeenLinesFormatError extends Error {
+  constructor(message) {
+    super(message);
+    this.name = 'SeenLinesFormatError';
+  }
+}
+
+function lineAt(lines, index) {
+  return lines.subarray(index * LINE_BYTES, (index + 1) * LINE_BYTES);
+}
+
+// The lines of a seen-lines file, packed 8 bytes each and sorted, after checking the header, the
+// length and the order. Throws a SeenLinesFormatError for anything else.
+export function decodeSeenLines(bytes) {
+  if (bytes.length < HEADER_BYTES || !bytes.subarray(0, MAGIC.length).equals(MAGIC)) {
+    throw new SeenLinesFormatError('it does not start as a KitCode seen-lines file does');
+  }
+
+  const count = bytes.readBigUInt64BE(MAGIC.length);
+  const held = bytes.length - HEADER_BYTES;
+
+  if (BigInt(held) !== count * BigInt(LINE_BYTES)) {
+    throw new SeenLinesFormatError(`its header names ${count} lines, and it holds ${held} bytes of lines`);
+  }
+
+  const lines = bytes.subarray(HEADER_BYTES);
+
+  for (let index = 1; index < held / LINE_BYTES; index += 1) {
+    if (Buffer.compare(lineAt(lines, index - 1), lineAt(lines, index)) >= 0) {
+      throw new SeenLinesFormatError(`its lines are not in ascending order at line ${index + 1}`);
+    }
+  }
+
+  return lines;
+}
+
+export function encodeSeenLines(lines) {
+  const header = Buffer.alloc(HEADER_BYTES);
+  MAGIC.copy(header);
+  header.writeBigUInt64BE(BigInt(lines.length / LINE_BYTES), MAGIC.length);
+  return Buffer.concat([header, lines]);
+}
+
+// The first line index at or after `from` whose line is not below `bytes`.
+function lowerBound(lines, bytes, from = 0) {
+  let low = from;
+  let high = lines.length / LINE_BYTES;
+
+  while (low < high) {
+    const middle = (low + high) >>> 1;
+
+    if (Buffer.compare(lineAt(lines, middle), bytes) < 0) {
+      low = middle + 1;
+    } else {
+      high = middle;
+    }
+  }
+
+  return low;
+}
+
+function keyBytes(key) {
+  if (!LINE_KEY.test(key)) {
+    throw new TypeError(`${JSON.stringify(key)} is not a line key of 16 hexadecimal characters`);
+  }
+
+  return Buffer.from(key, 'hex');
+}
+
+// Whether the line with snapshot key `key` is among `lines`.
+export function hasSeenLine(lines, key) {
+  const bytes = keyBytes(key);
+  const index = lowerBound(lines, bytes);
+  return index < lines.length / LINE_BYTES && lineAt(lines, index).equals(bytes);
+}
+
+// `lines` with the lines of `keys` among them, still sorted and each once. When every one is there
+// already, `lines` itself comes back, so the caller can tell nothing was added.
+export function addSeenLines(lines, keys) {
+  const added = [...new Set(keys)].sort().map(keyBytes).filter((bytes) => {
+    const index = lowerBound(lines, bytes);
+    return !(index < lines.length / LINE_BYTES && lineAt(lines, index).equals(bytes));
+  });
+
+  if (added.length === 0) {
+    return lines;
+  }
+
+  const merged = Buffer.alloc(lines.length + added.length * LINE_BYTES);
+  let copied = 0;
+  let written = 0;
+
+  for (const bytes of added) {
+    const index = lowerBound(lines, bytes, copied);
+    written += lines.copy(merged, written, copied * LINE_BYTES, index * LINE_BYTES);
+    written += bytes.copy(merged, written);
+    copied = index;
+  }
+
+  lines.copy(merged, written, copied * LINE_BYTES);
+  return merged;
+}
+
+// The lines in the file at `filePath`, or null when there is none. Throws a SeenLinesFormatError when
+// the file is not a seen-lines file, and the read's own error when it cannot be read.
+export function readSeenLinesFile(filePath) {
+  let bytes;
+
+  try {
+    bytes = fs.readFileSync(filePath);
+  } catch (error) {
+    if (error.code === 'ENOENT') {
+      return null;
+    }
+
+    throw error;
+  }
+
+  return decodeSeenLines(bytes);
+}
+
+// Writes `lines` to `filePath` atomically, 0600. Only under the state lock: a temporary file left
+// next to it then belongs to a writer that died.
+export function writeSeenLinesFile(filePath, lines) {
+  for (const tempPath of leftoverTempFiles(filePath)) {
+    fs.rmSync(tempPath, {force: true});
+  }
+
+  writeFileAtomically(filePath, encodeSeenLines(lines), {mode: FILE_MODE});
+}

package/src/skill-installer.mjs

@@ -0,0 +1,76 @@
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import {displayHomePath, kitCodeSkillMarkdown, sourceConfig} from './integration-spec.mjs';
+
+function skillPath(source) {
+  return path.join(os.homedir(), ...sourceConfig(source).skillPathParts);
+}
+
+function writeFileAtomic(filePath, value) {
+  fs.mkdirSync(path.dirname(filePath), {recursive: true});
+
+  if (fs.existsSync(filePath)) {
+    const backupPath = `${filePath}.kitcode.bak`;
+
+    if (!fs.existsSync(backupPath)) {
+      fs.copyFileSync(filePath, backupPath);
+    }
+  }
+
+  const tmpPath = `${filePath}.${process.pid}.tmp`;
+  fs.writeFileSync(tmpPath, value);
+  fs.renameSync(tmpPath, filePath);
+}
+
+export function installSkill(source) {
+  const filePath = skillPath(source);
+  writeFileAtomic(filePath, kitCodeSkillMarkdown(source));
+
+  return {
+    installed: true,
+    path: filePath,
+  };
+}
+
+export function uninstallSkill(source) {
+  const filePath = skillPath(source);
+  const removed = fs.existsSync(filePath);
+
+  if (removed) {
+    fs.rmSync(filePath);
+  }
+
+  return {
+    installed: false,
+    removed,
+    path: filePath,
+  };
+}
+
+export function uninstallSkillTree(source) {
+  const filePath = skillPath(source);
+  const skillDir = path.dirname(filePath);
+  const removed = fs.existsSync(skillDir);
+
+  if (removed) {
+    fs.rmSync(skillDir, {recursive: true, force: true});
+  }
+
+  return {
+    removed,
+    path: skillDir,
+  };
+}
+
+export function skillStatus(source) {
+  const filePath = skillPath(source);
+
+  return {
+    installed: fs.existsSync(filePath),
+    path: filePath,
+    displayPath: displayHomePath(sourceConfig(source).skillPathParts),
+  };
+}
+
+export {kitCodeSkillMarkdown};