gnutella 1.0.0 → 1.1.0

package/src/protocol/qrp.ts

@@ -1,549 +1 @@
-import crypto from "node:crypto";
-import fs from "node:fs";
-import zlib from "node:zlib";
-
-import {
-  DEFAULT_QRP_ENTRY_BITS,
-  DEFAULT_QRP_INFINITY,
-  DEFAULT_QRP_TABLE_SIZE,
-  QRP_COMPRESSOR_DEFLATE,
-  QRP_COMPRESSOR_NONE,
-  QRP_HASH_MULTIPLIER,
-} from "../const";
-import type { QueryDescriptor, RemoteQrpState, ShareFile } from "../types";
-import { base32Encode } from "./content_urn";
-
-type QrpPatchMessage = {
-  seqNo: number;
-  seqSize: number;
-  compressor: number;
-  entryBits: number;
-};
-
-type QrpResetMessage = {
-  tableLength: number;
-  infinity: number;
-};
-
-const QRP_MIN_WORD_LENGTH = 3;
-const QRP_MAX_CUT_CHARS = 5;
-
-export function splitSearchTerms(input: string): string[] {
-  const ascii = input
-    .normalize("NFKD")
-    .replace(/[\u0300-\u036f]/g, "")
-    .toLowerCase();
-  return ascii.split(/[^a-z0-9]+/).filter(Boolean);
-}
-
-export function tokenizeKeywords(input: string): string[] {
-  return [...new Set(splitSearchTerms(input).filter((x) => x.length > 1))];
-}
-
-function qrpQueryTerms(input: string): string[] {
-  return [
-    ...new Set(
-      splitSearchTerms(input).filter(
-        (x) => x.length >= QRP_MIN_WORD_LENGTH,
-      ),
-    ),
-  ];
-}
-
-function qrpIndexTerms(input: string): string[] {
-  const out: string[] = [];
-  const seen = new Set<string>();
-  for (const term of qrpQueryTerms(input)) {
-    let candidate = term;
-    for (let trim = 0; trim <= QRP_MAX_CUT_CHARS; trim++) {
-      if (!seen.has(candidate)) {
-        seen.add(candidate);
-        out.push(candidate);
-      }
-      if (candidate.length <= QRP_MIN_WORD_LENGTH) break;
-      const next = candidate.slice(0, -1);
-      if (next.length <= QRP_MIN_WORD_LENGTH) break;
-      candidate = next;
-    }
-  }
-  return out;
-}
-
-function qrpWordHitThreshold(hit: number, word: number): boolean {
-  return word < 3 ? hit === word : Math.trunc((3 * hit) / word) >= 2;
-}
-
-function qrpTermsMatch(
-  terms: string[],
-  hasTerm: (term: string) => boolean,
-): boolean {
-  if (!terms.length) return true;
-  let hit = 0;
-  for (const term of terms) {
-    if (hasTerm(term)) hit++;
-  }
-  return qrpWordHitThreshold(hit, terms.length);
-}
-
-function encodeSignedPatchValue(delta: number, bits: number): number {
-  const signBit = 1 << (bits - 1);
-  const min = -signBit;
-  const max = signBit - 1;
-  if (delta < min || delta > max)
-    throw new Error(`QRP ${bits}-bit patch delta out of range ${delta}`);
-  return delta & ((1 << bits) - 1);
-}
-
-function applyPresencePatchValue(
-  current: number,
-  infinity: number,
-  encoded: number,
-  bits: number,
-): number {
-  if (encoded === 0) return current;
-  const signBit = 1 << (bits - 1);
-  return encoded & signBit ? 1 : infinity;
-}
-
-function flipPresencePatchValue(
-  current: number,
-  infinity: number,
-): number {
-  return current < infinity ? infinity : 1;
-}
-
-function qrpHashBytes(str: string): number[] {
-  const bytes: number[] = [];
-  for (const char of str) {
-    let codePoint = char.codePointAt(0) ?? 0;
-    if (codePoint >= 0x41 && codePoint <= 0x5a) codePoint += 0x20;
-    if (codePoint <= 0xffff) {
-      bytes.push(codePoint & 0xff);
-      continue;
-    }
-    const value = codePoint - 0x10000;
-    const high = 0xd800 + (value >> 10);
-    const low = 0xdc00 + (value & 0x3ff);
-    bytes.push(high & 0xff, low & 0xff);
-  }
-  return bytes;
-}
-
-function qrpHash(str: string, bits: number): number {
-  const bytes = qrpHashBytes(str);
-  let xor = 0;
-  for (let i = 0; i < bytes.length; i++) xor ^= bytes[i] << ((i & 3) * 8);
-  const prod = BigInt(xor >>> 0) * BigInt(QRP_HASH_MULTIPLIER >>> 0);
-  const mask = (1n << BigInt(bits)) - 1n;
-  return Number((prod >> BigInt(32 - bits)) & mask) >>> 0;
-}
-
-export function sha1ToUrn(sha1: Buffer): string {
-  return `urn:sha1:${base32Encode(sha1)}`;
-}
-
-export async function sha1File(abs: string): Promise<Buffer> {
-  return await new Promise<Buffer>((resolve, reject) => {
-    const hash = crypto.createHash("sha1");
-    const rs = fs.createReadStream(abs);
-    rs.on("data", (chunk) => hash.update(chunk));
-    rs.on("error", reject);
-    rs.on("end", () => resolve(hash.digest()));
-  });
-}
-
-export function initialRemoteQrpState(): RemoteQrpState {
-  return {
-    resetSeen: false,
-    tableSize: 0,
-    infinity: DEFAULT_QRP_INFINITY,
-    entryBits: DEFAULT_QRP_ENTRY_BITS,
-    table: null,
-    seqSize: 0,
-    compressor: QRP_COMPRESSOR_NONE,
-    parts: new Map<number, Buffer>(),
-  };
-}
-
-export function validateRemoteQrpPatchSequence(
-  state: RemoteQrpState,
-  msg: QrpPatchMessage,
-): string | undefined {
-  return (
-    validateQrpResetSeen(state) ??
-    validateQrpPatchBounds(msg) ??
-    validateQrpPatchOrder(state, msg) ??
-    validateQrpPatchCodec(msg) ??
-    validateQrpPatchStability(state, msg)
-  );
-}
-
-export function validateRemoteQrpReset(
-  msg: QrpResetMessage,
-): string | undefined {
-  if (!isPowerOfTwo(msg.tableLength))
-    return `Invalid QRP table length ${msg.tableLength}`;
-  if (msg.infinity < 1) return `Invalid QRP infinity ${msg.infinity}`;
-  return undefined;
-}
-
-function isPowerOfTwo(value: number): boolean {
-  return value > 0 && Number.isInteger(Math.log2(value));
-}
-
-function validateQrpResetSeen(state: RemoteQrpState): string | undefined {
-  if (!state.resetSeen) return "No QRP RESET received before PATCH";
-  return undefined;
-}
-
-function validateQrpPatchBounds(msg: QrpPatchMessage): string | undefined {
-  if (msg.seqSize < 1 || msg.seqNo < 1 || msg.seqNo > msg.seqSize)
-    return `Invalid QRP seq number ${msg.seqNo} of ${msg.seqSize}`;
-  return undefined;
-}
-
-function validateQrpPatchOrder(
-  state: RemoteQrpState,
-  msg: QrpPatchMessage,
-): string | undefined {
-  const expectedSeqNo = state.seqSize ? state.parts.size + 1 : 1;
-  if (msg.seqNo !== expectedSeqNo)
-    return `Invalid QRP seq number ${msg.seqNo} (expected ${expectedSeqNo})`;
-  if (state.seqSize && msg.seqSize !== state.seqSize)
-    return `Changed QRP seq size to ${msg.seqSize} at message #${msg.seqNo} (began with ${state.seqSize})`;
-  return undefined;
-}
-
-function validateQrpPatchCodec(msg: QrpPatchMessage): string | undefined {
-  if (
-    msg.compressor !== QRP_COMPRESSOR_NONE &&
-    msg.compressor !== QRP_COMPRESSOR_DEFLATE
-  )
-    return `Invalid QRP compressor ${msg.compressor}`;
-
-  if (![1, 2, 4, 8].includes(msg.entryBits))
-    return `Invalid QRP entry bits ${msg.entryBits}`;
-  return undefined;
-}
-
-function validateQrpPatchStability(
-  state: RemoteQrpState,
-  msg: QrpPatchMessage,
-): string | undefined {
-  if (state.seqSize && msg.entryBits !== state.entryBits)
-    return `Changed QRP patch entry bits to ${msg.entryBits} at message #${msg.seqNo} (began with ${state.entryBits})`;
-
-  if (state.seqSize && msg.compressor !== state.compressor)
-    return `Changed QRP compressor to ${msg.compressor} at message #${msg.seqNo} (began with ${state.compressor})`;
-
-  return undefined;
-}
-
-export class QrpTable {
-  tableSize: number;
-  infinity: number;
-  entryBits: number;
-  table: Uint8Array;
-
-  constructor(
-    tableSize = DEFAULT_QRP_TABLE_SIZE,
-    infinity = DEFAULT_QRP_INFINITY,
-    entryBits = DEFAULT_QRP_ENTRY_BITS,
-  ) {
-    this.tableSize = tableSize;
-    this.infinity = infinity;
-    this.entryBits = entryBits;
-    this.table = new Uint8Array(tableSize);
-    this.clear();
-  }
-
-  clear(): void {
-    this.table.fill(this.infinity);
-  }
-
-  rebuildFromShares(shares: ShareFile[]): void {
-    this.clear();
-    for (const share of shares) {
-      for (const kw of share.keywords) {
-        for (const term of qrpIndexTerms(kw))
-          this.table[this.hashKeyword(term)] = 1;
-      }
-    }
-  }
-
-  hashKeyword(keyword: string): number {
-    return qrpHash(keyword, Math.log2(this.tableSize));
-  }
-
-  matchesQuery(search: string): boolean {
-    const kws = qrpQueryTerms(search);
-    return qrpTermsMatch(
-      kws,
-      (kw) => this.table[this.hashKeyword(kw)] < this.infinity,
-    );
-  }
-
-  mergePresenceTable(table: Uint8Array, infinity: number): void {
-    const length = Math.min(this.table.length, table.length);
-    for (let i = 0; i < length; i++) {
-      if (table[i] < infinity) this.table[i] = 1;
-    }
-  }
-
-  mergeFromQrp(other: Pick<QrpTable, "table" | "infinity">): void {
-    this.mergePresenceTable(other.table, other.infinity);
-  }
-
-  mergeFromRemoteQrp(state: RemoteQrpState): void {
-    if (!state.table) return;
-    this.mergePresenceTable(state.table, state.infinity);
-  }
-
-  encodeReset(): Buffer {
-    const payload = Buffer.alloc(6);
-    payload[0] = 0x00;
-    payload.writeUInt32LE(this.tableSize, 1);
-    payload[5] = this.infinity;
-    return payload;
-  }
-
-  encodePatchChunks(maxChunkPayload: number): Buffer[] {
-    const packed = this.packTable();
-    const compressed = zlib.deflateSync(packed);
-    const chunks: Buffer[] = [];
-    const partSize = Math.max(256, maxChunkPayload - 5);
-    const parts: Buffer[] = [];
-    for (let off = 0; off < compressed.length; off += partSize)
-      parts.push(compressed.subarray(off, off + partSize));
-    for (let i = 0; i < parts.length; i++) {
-      const payload = Buffer.alloc(5 + parts[i].length);
-      payload[0] = 0x01;
-      payload[1] = i + 1;
-      payload[2] = parts.length;
-      payload[3] = QRP_COMPRESSOR_DEFLATE;
-      payload[4] = this.entryBits;
-      parts[i].copy(payload, 5);
-      chunks.push(payload);
-    }
-    return chunks;
-  }
-
-  packTable(): Buffer {
-    if (this.entryBits === 1) return this.packOneBitTable();
-    if (this.entryBits === 4) return this.packNibbleTable();
-    if (this.entryBits === 8) return this.packByteTable();
-    throw new Error(`unsupported QRP entry bits ${this.entryBits}`);
-  }
-
-  packOneBitTable(): Buffer {
-    const out = Buffer.alloc(Math.ceil(this.tableSize / 8));
-    for (let i = 0; i < this.tableSize; i++) {
-      const byteIdx = i >> 3;
-      const bit = 7 - (i & 7);
-      if (this.table[i] < this.infinity) out[byteIdx] |= 1 << bit;
-    }
-    return out;
-  }
-
-  packNibbleTable(): Buffer {
-    const out = Buffer.alloc(Math.ceil(this.tableSize / 2));
-    for (let i = 0; i < this.tableSize; i++) {
-      const delta =
-        this.table[i] < this.infinity ? this.table[i] - this.infinity : 0;
-      const nibble = encodeSignedPatchValue(delta, 4);
-      const byteIdx = i >> 1;
-      if ((i & 1) === 0)
-        out[byteIdx] = (out[byteIdx] & 0x0f) | (nibble << 4);
-      else out[byteIdx] = (out[byteIdx] & 0xf0) | nibble;
-    }
-    return out;
-  }
-
-  packByteTable(): Buffer {
-    const out = Buffer.alloc(this.tableSize);
-    for (let i = 0; i < this.tableSize; i++) {
-      const delta =
-        this.table[i] < this.infinity ? this.table[i] - this.infinity : 0;
-      out[i] = encodeSignedPatchValue(delta, 8);
-    }
-    return out;
-  }
-
-  static canApplyPatch(state: RemoteQrpState): boolean {
-    return (
-      state.resetSeen &&
-      state.parts.size === state.seqSize &&
-      state.seqSize > 0
-    );
-  }
-
-  static orderedPatchParts(state: RemoteQrpState): Buffer[] | undefined {
-    const rawParts: Buffer[] = [];
-    for (let i = 1; i <= state.seqSize; i++) {
-      const part = state.parts.get(i);
-      if (!part) return undefined;
-      rawParts.push(part);
-    }
-    return rawParts;
-  }
-
-  static createUnpackedTable(state: RemoteQrpState): Uint8Array {
-    const table = new Uint8Array(state.tableSize);
-    table.fill(state.infinity);
-    return table;
-  }
-
-  static mutablePatchTable(state: RemoteQrpState): Uint8Array {
-    return state.table?.slice() ?? QrpTable.createUnpackedTable(state);
-  }
-
-  static unpackOneBitTable(
-    state: RemoteQrpState,
-    packed: Buffer,
-  ): Uint8Array {
-    const table = QrpTable.mutablePatchTable(state);
-    for (let i = 0; i < state.tableSize; i++) {
-      const byteIdx = i >> 3;
-      const bit = 7 - (i & 7);
-      if (byteIdx < packed.length && packed[byteIdx] & (1 << bit))
-        table[i] = flipPresencePatchValue(table[i], state.infinity);
-    }
-    return table;
-  }
-
-  static unpackNibbleTable(
-    state: RemoteQrpState,
-    packed: Buffer,
-  ): Uint8Array {
-    const table = QrpTable.mutablePatchTable(state);
-    for (let i = 0; i < state.tableSize; i++) {
-      const byteIdx = i >> 1;
-      if (byteIdx >= packed.length) break;
-      const nibble =
-        (i & 1) === 0
-          ? (packed[byteIdx] >> 4) & 0x0f
-          : packed[byteIdx] & 0x0f;
-      table[i] = applyPresencePatchValue(
-        table[i],
-        state.infinity,
-        nibble,
-        4,
-      );
-    }
-    return table;
-  }
-
-  static unpackByteTable(
-    state: RemoteQrpState,
-    packed: Buffer,
-  ): Uint8Array {
-    const table = QrpTable.mutablePatchTable(state);
-    for (let i = 0; i < state.tableSize; i++) {
-      if (i >= packed.length) break;
-      table[i] = applyPresencePatchValue(
-        table[i],
-        state.infinity,
-        packed[i],
-        8,
-      );
-    }
-    return table;
-  }
-
-  static unpackRemoteTable(
-    state: RemoteQrpState,
-    packed: Buffer,
-  ): Uint8Array | undefined {
-    if (state.entryBits === 1)
-      return QrpTable.unpackOneBitTable(state, packed);
-    if (state.entryBits === 4)
-      return QrpTable.unpackNibbleTable(state, packed);
-    if (state.entryBits === 8)
-      return QrpTable.unpackByteTable(state, packed);
-    return undefined;
-  }
-
-  static expectedPackedPatchBytes(
-    state: RemoteQrpState,
-  ): number | undefined {
-    if (state.entryBits === 1) return Math.ceil(state.tableSize / 8);
-    if (state.entryBits === 4) return Math.ceil(state.tableSize / 2);
-    if (state.entryBits === 8) return state.tableSize;
-    return undefined;
-  }
-
-  static packedPatchCoverageError(
-    state: RemoteQrpState,
-    packed: Buffer,
-  ): string | undefined {
-    const expectedBytes = QrpTable.expectedPackedPatchBytes(state);
-    if (expectedBytes == null || packed.length >= expectedBytes)
-      return undefined;
-    const coveredSlots = Math.floor((packed.length * 8) / state.entryBits);
-    return `Incomplete ${state.entryBits}-bit QRP patch covered ${coveredSlots}/${state.tableSize} slots`;
-  }
-
-  static applyPatch(state: RemoteQrpState): string | undefined {
-    if (!QrpTable.canApplyPatch(state)) return undefined;
-    const rawParts = QrpTable.orderedPatchParts(state);
-    if (!rawParts) return undefined;
-    let packed = Buffer.concat(rawParts);
-    if (state.compressor === QRP_COMPRESSOR_DEFLATE)
-      packed = zlib.inflateSync(packed);
-    const coverageError = QrpTable.packedPatchCoverageError(state, packed);
-    if (coverageError) return coverageError;
-    const table = QrpTable.unpackRemoteTable(state, packed);
-    if (!table) return undefined;
-    state.table = table;
-    state.parts.clear();
-    state.seqSize = 0;
-    return undefined;
-  }
-
-  static matchesRemote(state: RemoteQrpState, search: string): boolean {
-    if (!state.table || !state.tableSize) return true;
-    const kws = qrpQueryTerms(search);
-    const bits = Math.log2(state.tableSize);
-    return qrpTermsMatch(
-      kws,
-      (kw) => state.table![qrpHash(kw, bits)] < state.infinity,
-    );
-  }
-}
-
-export function matchQuery(
-  q: QueryDescriptor,
-  share: Pick<ShareFile, "name" | "sha1Urn" | "keywords">,
-): boolean {
-  if (q.urns.length) {
-    if (!share.sha1Urn) return false;
-    const urnSet = new Set(q.urns.map((x) => x.toLowerCase()));
-    if (!urnSet.has(share.sha1Urn.toLowerCase())) return false;
-  }
-  const term = q.search.trim();
-  if (!term) return q.urns.length > 0;
-  const kws = tokenizeKeywords(term);
-  if (!kws.length)
-    return share.name.toLowerCase().includes(term.toLowerCase());
-  const shareKw = new Set(share.keywords);
-  return kws.every(
-    (kw) => shareKw.has(kw) || share.name.toLowerCase().includes(kw),
-  );
-}
-
-function canRouteQrpQuery(
-  q: Pick<QueryDescriptor, "search" | "urns">,
-  matchesSearch: (search: string) => boolean,
-): boolean {
-  if (q.urns.length) return true;
-  return matchesSearch(q.search);
-}
-
-export function canRouteRemoteQrpQuery(
-  state: RemoteQrpState,
-  q: Pick<QueryDescriptor, "search" | "urns">,
-): boolean {
-  return canRouteQrpQuery(q, (search) =>
-    QrpTable.matchesRemote(state, search),
-  );
-}
+export { initialRemoteQrpState, QrpTable } from "../query_routing/qrp";

package/src/protocol/query_matching.ts

@@ -0,0 +1,22 @@
+import type { QueryDescriptor, ShareFile } from "../types";
+import { tokenizeKeywords } from "../query_routing/qrp";
+
+export function matchQuery(
+  q: QueryDescriptor,
+  share: Pick<ShareFile, "name" | "sha1Urn" | "keywords">,
+): boolean {
+  if (q.urns.length) {
+    if (!share.sha1Urn) return false;
+    const urnSet = new Set(q.urns.map((x) => x.toLowerCase()));
+    if (!urnSet.has(share.sha1Urn.toLowerCase())) return false;
+  }
+  const term = q.search.trim();
+  if (!term) return q.urns.length > 0;
+  const kws = tokenizeKeywords(term);
+  if (!kws.length)
+    return share.name.toLowerCase().includes(term.toLowerCase());
+  const shareKw = new Set(share.keywords);
+  return kws.every(
+    (kw) => shareKw.has(kw) || share.name.toLowerCase().includes(kw),
+  );
+}

package/src/protocol/share_index.ts

@@ -2,23 +2,16 @@ import fsp from "node:fs/promises";
 import path from "node:path";
 
 import { ensureDir, fileExists } from "../shared";
+import {
+  parseShareCatalogManifest,
+  serializeShareCatalogManifest,
+  type ShareCatalogEntry,
+} from "../share_catalog";
 
 const SHARE_INDEX_FILENAME = "share-index.json";
-const SHARE_INDEX_VERSION = 1;
 const shareIndexWriteQueues = new Map<string, Promise<void>>();
 
-export type ShareIndexEntry = {
-  rel: string;
-  size: number;
-  mtimeMs: number;
-  sha1Hex?: string;
-  sha1Urn?: string;
-};
-
-type ShareIndexManifest = {
-  version?: unknown;
-  files?: unknown;
-};
+export type ShareIndexEntry = ShareCatalogEntry;
 
 function shareIndexPath(dataDir: string): string {
   return path.join(dataDir, SHARE_INDEX_FILENAME);
@@ -44,50 +37,6 @@ async function queueShareIndexWrite(
   }
 }
 
-function objectRecord(
-  value: unknown,
-): Record<string, unknown> | undefined {
-  if (!value || typeof value !== "object" || Array.isArray(value))
-    return undefined;
-  return value as Record<string, unknown>;
-}
-
-function nonEmptyString(value: unknown): string | undefined {
-  return typeof value === "string" && value.length > 0 ? value : undefined;
-}
-
-function nonNegativeNumber(value: unknown): number | undefined {
-  const num = Number(value);
-  return Number.isFinite(num) && num >= 0 ? num : undefined;
-}
-
-function validSha1Hex(value: unknown): string | undefined {
-  const text = nonEmptyString(value);
-  return text && /^[0-9a-f]{40}$/i.test(text)
-    ? text.toLowerCase()
-    : undefined;
-}
-
-function parseShareIndexEntry(
-  value: unknown,
-): ShareIndexEntry | undefined {
-  const record = objectRecord(value);
-  if (!record) return undefined;
-  const rel = nonEmptyString(record.rel);
-  const size = nonNegativeNumber(record.size);
-  const mtimeMs = nonNegativeNumber(record.mtimeMs);
-  if (!rel || size == null || mtimeMs == null) return undefined;
-  const sha1Hex = validSha1Hex(record.sha1Hex);
-  const sha1Urn = nonEmptyString(record.sha1Urn);
-  return {
-    rel,
-    size,
-    mtimeMs,
-    ...(sha1Hex ? { sha1Hex } : {}),
-    ...(sha1Urn ? { sha1Urn } : {}),
-  };
-}
-
 export async function loadShareIndex(
   dataDir: string,
 ): Promise<Map<string, ShareIndexEntry>> {
@@ -95,13 +44,7 @@ export async function loadShareIndex(
   if (!(await fileExists(file))) return new Map();
   try {
     const raw = await fsp.readFile(file, "utf8");
-    const parsed = JSON.parse(raw) as ShareIndexManifest;
-    if (parsed.version !== SHARE_INDEX_VERSION) return new Map();
-    if (!Array.isArray(parsed.files)) return new Map();
-    const entries = parsed.files
-      .map((value) => parseShareIndexEntry(value))
-      .filter((entry) => entry !== undefined);
-    return new Map(entries.map((entry) => [entry.rel, entry]));
+    return parseShareCatalogManifest(JSON.parse(raw));
   } catch {
     return new Map();
   }
@@ -113,12 +56,7 @@ export async function writeShareIndex(
 ): Promise<void> {
   await ensureDir(dataDir);
   const file = shareIndexPath(dataDir);
-  const manifest = {
-    version: SHARE_INDEX_VERSION,
-    files: [...entries.values()].sort((a, b) =>
-      a.rel.localeCompare(b.rel),
-    ),
-  };
+  const manifest = serializeShareCatalogManifest(entries);
   await queueShareIndexWrite(file, async () => {
     const tmp = shareIndexTmpPath(file);
     try {