nosible 0.1.5

package/src/search/analyze.test.ts

@@ -0,0 +1,396 @@
+import {describe, expect, test} from "bun:test";
+import {analyzeResults, AnalyzeBy} from "./analyze";
+import {Result} from "./result";
+import {NosibleClient} from "../client";
+import {createMockClient} from "../test-utils/mocks";
+import type {SearchResponse} from "../api/schemas";
+
+describe("analyzeResults", () => {
+  const mockSearchResults: SearchResponse[] = [
+    {
+      url_hash: "hash1",
+      url: "https://example.com/article1",
+      netloc: "example.com",
+      published: "2023-01-15T10:00:00Z",
+      visited: "2023-01-16T12:00:00Z",
+      language: "en",
+      author: "John Doe",
+      title: "Test Article 1",
+      description: "Description 1",
+      content: "Content 1",
+      best_chunk: "Chunk 1",
+      semantics: {
+        origin_shard: 1,
+        chunks_total: 10,
+        chunks_matched: 5,
+        chunks_kept: 3,
+        similarity: 0.85,
+        bitstring: "101010",
+      },
+    },
+    {
+      url_hash: "hash2",
+      url: "https://test.com/article2",
+      netloc: "test.com",
+      published: "2023-02-20T14:30:00Z",
+      visited: "2023-02-21T16:00:00Z",
+      language: "en",
+      author: "Jane Smith",
+      title: "Test Article 2",
+      description: "Description 2",
+      content: "Content 2",
+      best_chunk: "Chunk 2",
+      semantics: {
+        origin_shard: 2,
+        chunks_total: 8,
+        chunks_matched: 4,
+        chunks_kept: 2,
+        similarity: 0.92,
+        bitstring: "110011",
+      },
+    },
+    {
+      url_hash: "hash3",
+      url: "https://example.com/article3",
+      netloc: "example.com",
+      published: "2023-03-10T09:15:00Z",
+      visited: "2023-03-11T11:30:00Z",
+      language: "fr",
+      author: "",
+      title: "Test Article 3",
+      description: "Description 3",
+      content: "Content 3",
+      best_chunk: "Chunk 3",
+      semantics: {
+        origin_shard: 1,
+        chunks_total: 12,
+        chunks_matched: 6,
+        chunks_kept: 4,
+        similarity: 0.78,
+        bitstring: "111000",
+      },
+    },
+  ];
+
+  // Convert SearchResponse objects to Result instances
+  const mockClient = createMockClient() as unknown as NosibleClient;
+  const mockResults = mockSearchResults.map(
+    (searchResponse) => new Result({client: mockClient, result: searchResponse})
+  );
+
+  test("analyzes by netloc", () => {
+    const result = analyzeResults(mockResults, AnalyzeBy.netloc);
+    expect(result).toEqual({
+      "example.com": 2,
+      "test.com": 1,
+    });
+  });
+
+  test("analyzes by published date", () => {
+    const result = analyzeResults(mockResults, AnalyzeBy.published);
+    expect(result).toEqual({
+      "2023-01": 1,
+      "2023-02": 1,
+      "2023-03": 1,
+    });
+  });
+
+  test("analyzes by visited date", () => {
+    const result = analyzeResults(mockResults, AnalyzeBy.visited);
+    expect(result).toEqual({
+      "2023-01": 1,
+      "2023-02": 1,
+      "2023-03": 1,
+    });
+  });
+
+  test("analyzes by author", () => {
+    const result = analyzeResults(mockResults, AnalyzeBy.author);
+    expect(result).toEqual({
+      "John Doe": 1,
+      "Jane Smith": 1,
+      "Unknown Author": 1,
+    });
+  });
+
+  test("analyzes by language", () => {
+    const result = analyzeResults(mockResults, AnalyzeBy.language);
+    expect(result).toEqual({
+      en: 2,
+      fr: 1,
+    });
+  });
+
+  test("analyzes by similarity", () => {
+    const result = analyzeResults(mockResults, AnalyzeBy.similarity);
+    expect(result).toHaveProperty("count", 3);
+    expect(result).toHaveProperty("null_count", 0);
+    expect(result).toHaveProperty("mean");
+    expect(result).toHaveProperty("std");
+    expect(result).toHaveProperty("min");
+    expect(result).toHaveProperty("q25");
+    expect(result).toHaveProperty("q50");
+    expect(result).toHaveProperty("q75");
+    expect(result).toHaveProperty("max");
+
+    // Check that the statistics are reasonable
+    expect((result as any).mean).toBeCloseTo(0.85, 1);
+    expect((result as any).min).toBe(0.78);
+    expect((result as any).max).toBe(0.92);
+  });
+
+  test("handles empty results array", () => {
+    const result = analyzeResults([], AnalyzeBy.netloc);
+    expect(result).toEqual({});
+  });
+
+  test("handles results with same month across different years", () => {
+    const resultsWithDifferentYears: SearchResponse[] = [
+      {
+        url_hash: "hash1",
+        url: "https://example.com/article1",
+        netloc: "example.com",
+        published: "2022-12-15T10:00:00Z",
+        visited: "2023-01-16T12:00:00Z",
+        language: "en",
+        author: "John Doe",
+        title: "Test Article 1",
+        description: "Description 1",
+        content: "Content 1",
+        best_chunk: "Chunk 1",
+        semantics: {
+          origin_shard: 1,
+          chunks_total: 10,
+          chunks_matched: 5,
+          chunks_kept: 3,
+          similarity: 0.85,
+          bitstring: "101010",
+        },
+      },
+      {
+        url_hash: "hash2",
+        url: "https://test.com/article2",
+        netloc: "test.com",
+        published: "2023-12-20T14:30:00Z",
+        visited: "2023-02-21T16:00:00Z",
+        language: "en",
+        author: "Jane Smith",
+        title: "Test Article 2",
+        description: "Description 2",
+        content: "Content 2",
+        best_chunk: "Chunk 2",
+        semantics: {
+          origin_shard: 2,
+          chunks_total: 8,
+          chunks_matched: 4,
+          chunks_kept: 2,
+          similarity: 0.92,
+          bitstring: "110011",
+        },
+      },
+    ];
+
+    // Convert to Result instances
+    const resultInstances = resultsWithDifferentYears.map(
+      (searchResponse) => new Result({client: mockClient, result: searchResponse})
+    );
+
+    const result = analyzeResults(
+      resultInstances,
+      AnalyzeBy.published
+    );
+    expect(result).toEqual({
+      "2022-12": 1,
+      "2023-01": 0,
+      "2023-02": 0,
+      "2023-03": 0,
+      "2023-04": 0,
+      "2023-05": 0,
+      "2023-06": 0,
+      "2023-07": 0,
+      "2023-08": 0,
+      "2023-09": 0,
+      "2023-10": 0,
+      "2023-11": 0,
+      "2023-12": 1,
+    });
+  });
+
+  test("handles results with missing author", () => {
+    const resultsWithMissingAuthor: SearchResponse[] = [
+      {
+        url_hash: "hash1",
+        url: "https://example.com/article1",
+        netloc: "example.com",
+        published: "2023-01-15T10:00:00Z",
+        visited: "2023-01-16T12:00:00Z",
+        language: "en",
+        author: "",
+        title: "Test Article 1",
+        description: "Description 1",
+        content: "Content 1",
+        best_chunk: "Chunk 1",
+        semantics: {
+          origin_shard: 1,
+          chunks_total: 10,
+          chunks_matched: 5,
+          chunks_kept: 3,
+          similarity: 0.85,
+          bitstring: "101010",
+        },
+      },
+      {
+        url_hash: "hash2",
+        url: "https://test.com/article2",
+        netloc: "test.com",
+        published: "2023-02-20T14:30:00Z",
+        visited: "2023-02-21T16:00:00Z",
+        language: "en",
+        author: "",
+        title: "Test Article 2",
+        description: "Description 2",
+        content: "Content 2",
+        best_chunk: "Chunk 2",
+        semantics: {
+          origin_shard: 2,
+          chunks_total: 8,
+          chunks_matched: 4,
+          chunks_kept: 2,
+          similarity: 0.92,
+          bitstring: "110011",
+        },
+      },
+    ];
+
+    // Convert to Result instances
+    const resultInstances = resultsWithMissingAuthor.map(
+      (searchResponse) => new Result({client: mockClient, result: searchResponse})
+    );
+
+    const result = analyzeResults(resultInstances, AnalyzeBy.author);
+    expect(result).toEqual({
+      "Unknown Author": 2,
+    });
+  });
+
+  test("handles similarity with null values", () => {
+    const resultsWithNullSimilarity: SearchResponse[] = [
+      {
+        url_hash: "hash1",
+        url: "https://example.com/article1",
+        netloc: "example.com",
+        published: "2023-01-15T10:00:00Z",
+        visited: "2023-01-16T12:00:00Z",
+        language: "en",
+        author: "John Doe",
+        title: "Test Article 1",
+        description: "Description 1",
+        content: "Content 1",
+        best_chunk: "Chunk 1",
+        semantics: {
+          origin_shard: 1,
+          chunks_total: 10,
+          chunks_matched: 5,
+          chunks_kept: 3,
+          similarity: 0.85,
+          bitstring: "101010",
+        },
+      },
+      {
+        url_hash: "hash2",
+        url: "https://test.com/article2",
+        netloc: "test.com",
+        published: "2023-02-20T14:30:00Z",
+        visited: "2023-02-21T16:00:00Z",
+        language: "en",
+        author: "Jane Smith",
+        title: "Test Article 2",
+        description: "Description 2",
+        content: "Content 2",
+        best_chunk: "Chunk 2",
+        semantics: {
+          origin_shard: 2,
+          chunks_total: 8,
+          chunks_matched: 4,
+          chunks_kept: 2,
+          similarity: null as any,
+          bitstring: "110011",
+        },
+      },
+    ];
+
+    // Convert to Result instances
+    const resultInstances = resultsWithNullSimilarity.map(
+      (searchResponse) => new Result({client: mockClient, result: searchResponse})
+    );
+
+    const result = analyzeResults(
+      resultInstances,
+      AnalyzeBy.similarity
+    );
+    expect((result as any).count).toBe(2);
+    expect((result as any).null_count).toBe(1);
+  });
+
+  test("handles invalid dates gracefully", () => {
+    const resultsWithInvalidDates: SearchResponse[] = [
+      {
+        url_hash: "hash1",
+        url: "https://example.com/article1",
+        netloc: "example.com",
+        published: "invalid-date",
+        visited: "2023-01-16T12:00:00Z",
+        language: "en",
+        author: "John Doe",
+        title: "Test Article 1",
+        description: "Description 1",
+        content: "Content 1",
+        best_chunk: "Chunk 1",
+        semantics: {
+          origin_shard: 1,
+          chunks_total: 10,
+          chunks_matched: 5,
+          chunks_kept: 3,
+          similarity: 0.85,
+          bitstring: "101010",
+        },
+      },
+      {
+        url_hash: "hash2",
+        url: "https://test.com/article2",
+        netloc: "test.com",
+        published: "2023-02-20T14:30:00Z",
+        visited: "2023-02-21T16:00:00Z",
+        language: "en",
+        author: "Jane Smith",
+        title: "Test Article 2",
+        description: "Description 2",
+        content: "Content 2",
+        best_chunk: "Chunk 2",
+        semantics: {
+          origin_shard: 2,
+          chunks_total: 8,
+          chunks_matched: 4,
+          chunks_kept: 2,
+          similarity: 0.92,
+          bitstring: "110011",
+        },
+      },
+    ];
+
+    // Convert to Result instances
+    const resultInstances = resultsWithInvalidDates.map(
+      (searchResponse) => new Result({client: mockClient, result: searchResponse})
+    );
+
+    const result = analyzeResults(resultInstances, AnalyzeBy.published);
+    expect(result).toEqual({
+      "2023-02": 1,
+    });
+  });
+
+  test("handles default case", () => {
+    const result = analyzeResults(mockResults, 999 as AnalyzeBy);
+    expect(result).toEqual({});
+  });
+});

package/src/search/analyze.ts

@@ -0,0 +1,151 @@
+import {mean, standardDeviation, min, max, quantile} from "simple-statistics";
+import type {Result} from "./result";
+
+export enum AnalyzeBy {
+  netloc,
+  published,
+  visited,
+  author,
+  language,
+  similarity,
+}
+
+export type AnalyzeResult =
+  | AnalyzeNumericResult
+  | AnalyzeCategoricalResult
+  | AnalyzeDateResult;
+
+export const analyzeResults = (
+  results: Result[],
+  by: AnalyzeBy
+): AnalyzeResult => {
+  switch (by) {
+    case AnalyzeBy.netloc:
+      const netlocValues = results.map((r) => r.netloc);
+      const netlocStats = analyzeCategorical(netlocValues);
+      return netlocStats;
+    case AnalyzeBy.published:
+      const publishedValues = results.map((r) => new Date(r.published));
+      const publishedStats = analyzeDateByMonth(publishedValues);
+      return publishedStats;
+    case AnalyzeBy.visited:
+      const visitedValues = results.map((r) => new Date(r.visited));
+      const visitedStats = analyzeDateByMonth(visitedValues);
+      return visitedStats;
+    case AnalyzeBy.author:
+      const authorValues = results.map((r) => {
+        if (r.author) {
+          return r.author;
+        } else {
+          return "Unknown Author";
+        }
+      });
+      const authorStats = analyzeCategorical(authorValues);
+      return authorStats;
+    case AnalyzeBy.language:
+      const languageValues = results.map((r) => r.language);
+      const languageStats = analyzeCategorical(languageValues);
+      return languageStats;
+    case AnalyzeBy.similarity:
+      const simValues = results.map((r) => r.semantics.similarity);
+      const simStats = analyzeNumeric(simValues);
+      return simStats;
+    default:
+      return {};
+  }
+};
+
+type AnalyzeCategoricalResult = Record<string, number>;
+
+const analyzeCategorical = (arr: string[]): AnalyzeCategoricalResult => {
+  const result: Record<string, number> = {};
+  for (const v of arr) {
+    if (result[v]) {
+      result[v] += 1;
+    } else {
+      result[v] = 1;
+    }
+  }
+  return result;
+};
+
+type AnalyzeNumericResult = {
+  count: number;
+  null_count: number;
+  mean: number;
+  std: number;
+  min: number;
+  q25: number;
+  q50: number;
+  q75: number;
+  max: number;
+};
+
+const analyzeNumeric = (arr: number[]): AnalyzeNumericResult => {
+  const meanValue = mean(arr);
+  const stdValue = standardDeviation(arr);
+  const minValue = min(arr);
+  const maxValue = max(arr);
+  const q25 = quantile(arr, 0.25);
+  const q50 = quantile(arr, 0.5);
+  const q75 = quantile(arr, 0.75);
+
+  // TODO: confirm quantile to percentiles
+
+  return {
+    count: arr.length,
+    null_count: arr.filter((v) => v === null).length,
+    mean: meanValue,
+    std: stdValue,
+    min: minValue,
+    q25: q25,
+    q50: q50,
+    q75: q75,
+    max: maxValue,
+  };
+};
+
+type AnalyzeDateResult = Record<string, number>;
+
+const analyzeDateByMonth = (arr: Date[]): AnalyzeDateResult => {
+  if (!arr || arr.length === 0) return {};
+
+  let minDate: Date | null = null;
+  let maxDate: Date | null = null;
+  const counts: Record<string, number> = {};
+
+  for (const d of arr) {
+    if (!(d instanceof Date)) continue;
+    const time = d.getTime();
+    if (Number.isNaN(time)) continue;
+
+    if (!minDate || d < minDate) minDate = d;
+    if (!maxDate || d > maxDate) maxDate = d;
+
+    const key = `${d.getFullYear()}-${String(d.getMonth() + 1).padStart(
+      2,
+      "0"
+    )}`;
+    counts[key] = (counts[key] ?? 0) + 1;
+  }
+
+  if (!minDate || !maxDate) return {};
+
+  const result: Record<string, number> = {};
+  let y = minDate.getFullYear();
+  let m = minDate.getMonth() + 1;
+  const endY = maxDate.getFullYear();
+  const endM = maxDate.getMonth() + 1;
+
+  while (y < endY || (y === endY && m <= endM)) {
+    const key = `${y}-${String(m).padStart(2, "0")}`;
+    result[key] = counts[key] ?? 0;
+    m += 1;
+    if (m > 12) {
+      m = 1;
+      y += 1;
+    }
+  }
+
+  return result;
+};

package/src/search/bulkSearch.ts

@@ -0,0 +1,62 @@
+import {gunzipSync} from "zlib";
+import {fernetDecrypt} from "../utils/fernet";
+
+/**
+ * Download and decrypt bulk search results
+ * @param data - Bulk search response containing download_from and decrypt_using
+ * @param filterResponses - Maximum number of results to return
+ * @returns ResultSet containing the search results
+ */
+export async function bulkSearchDownload({
+  downloadFrom,
+  decryptUsing,
+}: {
+  downloadFrom: string;
+  decryptUsing: string;
+}): Promise<any> {
+  // Replace .zstd. with .gzip. in download URL
+  if (downloadFrom.includes(".zstd.")) {
+    downloadFrom = downloadFrom.replace(".zstd.", ".gzip.");
+  } else {
+    throw new Error("Results were not retrieved from Nosible");
+  }
+
+  // Retry up to 100 times with 10 second delays
+  try {
+    for (let attempt = 0; attempt < 100; attempt++) {
+      const response = await fetch(downloadFrom);
+
+      if (response.status === 200) {
+        try {
+          const responseData = await response.arrayBuffer();
+          const encryptedData = Buffer.from(responseData);
+
+          // Fernet tokens are base64-encoded, so decode first
+          const tokenBuffer = Buffer.from(
+            encryptedData.toString("utf-8"),
+            "base64"
+          );
+
+          // Decrypt using Fernet
+          const decrypted = fernetDecrypt(tokenBuffer, decryptUsing);
+
+          // Decompress using gzip
+          const decompressed = gunzipSync(decrypted);
+
+          // Parse JSON
+          const apiResp = JSON.parse(decompressed.toString("utf-8"));
+
+          return apiResp;
+        } catch (error) {
+          console.error("Error decrypting data:", error);
+          break;
+        }
+      }
+      // Wait 10 seconds before retrying
+      await new Promise((resolve) => setTimeout(resolve, 10000));
+    }
+  } catch (error) {
+    console.error("Error downloading data:", error);
+    throw error;
+  }
+}