nosible 0.1.5

package/src/search/sqlFilter.test.ts

@@ -0,0 +1,129 @@
+import {describe, expect, test} from "bun:test";
+import {generateSqlFilter} from "./sqlFilter";
+
+describe("generateSqlFilter", () => {
+  test("generates basic SQL without filters", () => {
+    const sql = generateSqlFilter({});
+    expect(sql).toBe(null);
+  });
+
+  test("includes published date range", () => {
+    const start = new Date("2023-01-01T00:00:00Z");
+    const end = new Date("2023-12-31T23:59:59Z");
+    const sql = generateSqlFilter({
+      publishStart: start,
+      publishEnd: end,
+    });
+    expect(sql).toBe(
+      "SELECT loc FROM engine WHERE published >= '2023-01-01T00:00:00.000Z' AND published <= '2023-12-31T23:59:59.000Z'"
+    );
+  });
+
+  test("includes published start only", () => {
+    const start = new Date("2023-01-01T00:00:00Z");
+    const sql = generateSqlFilter({publishStart: start});
+    expect(sql).toBe(
+      "SELECT loc FROM engine WHERE published >= '2023-01-01T00:00:00.000Z'"
+    );
+  });
+
+  test("includes published end only", () => {
+    const end = new Date("2023-12-31T23:59:59Z");
+    const sql = generateSqlFilter({publishEnd: end});
+    expect(sql).toBe(
+      "SELECT loc FROM engine WHERE published <= '2023-12-31T23:59:59.000Z'"
+    );
+  });
+
+  test("includes visited date range", () => {
+    const start = new Date("2023-01-01T00:00:00Z");
+    const end = new Date("2023-12-31T23:59:59Z");
+    const sql = generateSqlFilter({
+      visitedStart: start,
+      visitedEnd: end,
+    });
+    expect(sql).toBe(
+      "SELECT loc FROM engine WHERE visited >= '2023-01-01T00:00:00.000Z' AND visited <= '2023-12-31T23:59:59.000Z'"
+    );
+  });
+
+  test("includes certain true", () => {
+    const sql = generateSqlFilter({certain: true});
+    expect(sql).toBe("SELECT loc FROM engine WHERE certain = TRUE");
+  });
+
+  test("includes certain false", () => {
+    const sql = generateSqlFilter({certain: false});
+    expect(sql).toBe("SELECT loc FROM engine WHERE certain = FALSE");
+  });
+
+  test("includes netlocs with variants", () => {
+    const sql = generateSqlFilter({
+      includeNetlocs: ["example.com", "www.test.com"],
+    });
+    expect(sql).toBe(
+      "SELECT loc FROM engine WHERE netloc IN ('example.com', 'test.com', 'www.example.com', 'www.test.com')"
+    );
+  });
+
+  test("excludes netlocs with variants", () => {
+    const sql = generateSqlFilter({excludeNetlocs: ["example.com"]});
+    expect(sql).toBe(
+      "SELECT loc FROM engine WHERE netloc NOT IN ('example.com', 'www.example.com')"
+    );
+  });
+
+  test("includes companies", () => {
+    const sql = generateSqlFilter({includeCompanies: ["GOOG", "AAPL"]});
+    expect(sql).toBe(
+      "SELECT loc FROM engine WHERE (companies IS NOT NULL AND (ARRAY_CONTAINS(companies, 'GOOG') OR ARRAY_CONTAINS(companies, 'AAPL')))"
+    );
+  });
+
+  test("excludes companies", () => {
+    const sql = generateSqlFilter({excludeCompanies: ["GOOG"]});
+    expect(sql).toBe(
+      "SELECT loc FROM engine WHERE (companies IS NULL OR NOT (ARRAY_CONTAINS(companies, 'GOOG')))"
+    );
+  });
+
+  test("includes docs", () => {
+    const sql = generateSqlFilter({includeDocs: ["hash1", "hash2"]});
+    expect(sql).toBe(
+      "SELECT loc FROM engine WHERE doc_hash IN ('hash1', 'hash2')"
+    );
+  });
+
+  test("excludes docs", () => {
+    const sql = generateSqlFilter({excludeDocs: ["hash1"]});
+    expect(sql).toBe("SELECT loc FROM engine WHERE doc_hash NOT IN ('hash1')");
+  });
+
+  test("combines multiple filters", () => {
+    const sql = generateSqlFilter({
+      publishStart: new Date("2023-01-01T00:00:00Z"),
+      certain: true,
+      includeNetlocs: ["example.com"],
+      includeCompanies: ["GOOG"],
+    });
+    expect(sql).toContain("published >= '2023-01-01T00:00:00.000Z'");
+    expect(sql).toContain("certain = TRUE");
+    expect(sql).toContain("netloc IN ('example.com', 'www.example.com')");
+    expect(sql).toContain(
+      "companies IS NOT NULL AND (ARRAY_CONTAINS(companies, 'GOOG'))"
+    );
+    expect(sql).toMatch(/^SELECT loc FROM engine WHERE/);
+  });
+
+  test("throws error for too many include_netlocs", () => {
+    const many = Array.from({length: 51}, (_, i) => `netloc${i}`);
+    expect(() => generateSqlFilter({includeNetlocs: many})).toThrow("Too big");
+  });
+
+  test("throws error for too many exclude_companies", () => {
+    const many = Array.from({length: 51}, (_, i) => `company${i}`);
+    expect(() => generateSqlFilter({excludeCompanies: many})).toThrow(
+      "Too big"
+    );
+  });
+});

package/src/search/sqlFilter.ts

@@ -0,0 +1,147 @@
+import z from "zod";
+
+export const sqlFilterSchema = z.object({
+  publishStart: z.date().optional(),
+  publishEnd: z.date().optional(),
+  visitedStart: z.date().optional(),
+  visitedEnd: z.date().optional(),
+  certain: z.boolean().optional(),
+  includeNetlocs: z.array(z.string()).max(50).optional(),
+  excludeNetlocs: z.array(z.string()).max(50).optional(),
+  includeCompanies: z.array(z.string()).max(50).optional(),
+  excludeCompanies: z.array(z.string()).max(50).optional(),
+  includeDocs: z.array(z.string()).max(50).optional(),
+  excludeDocs: z.array(z.string()).max(50).optional(),
+});
+
+export type SqlFilter = z.infer<typeof sqlFilterSchema>;
+
+export function generateSqlFilter(input: SqlFilter): string | null {
+  const validatedInput = sqlFilterSchema.parse(input);
+  const {
+    publishStart,
+    publishEnd,
+    visitedStart,
+    visitedEnd,
+    certain,
+    includeNetlocs,
+    excludeNetlocs,
+    includeCompanies,
+    excludeCompanies,
+    includeDocs,
+    excludeDocs,
+  } = validatedInput;
+  const sql: string[] = ["SELECT loc FROM engine"];
+  const clauses: string[] = [];
+
+  // Published date range
+  if (publishStart || publishEnd) {
+    if (publishStart && publishEnd) {
+      clauses.push(
+        `published >= '${publishStart.toISOString()}' AND published <= '${publishEnd.toISOString()}'`
+      );
+    } else if (publishStart) {
+      clauses.push(`published >= '${publishStart.toISOString()}'`);
+    } else if (publishEnd) {
+      clauses.push(`published <= '${publishEnd.toISOString()}'`);
+    }
+  }
+
+  // Visited date range
+  if (visitedStart || visitedEnd) {
+    if (visitedStart && visitedEnd) {
+      clauses.push(
+        `visited >= '${visitedStart.toISOString()}' AND visited <= '${visitedEnd.toISOString()}'`
+      );
+    } else if (visitedStart) {
+      clauses.push(`visited >= '${visitedStart.toISOString()}'`);
+    } else if (visitedEnd) {
+      clauses.push(`visited <= '${visitedEnd.toISOString()}'`);
+    }
+  }
+
+  // Date certainty filter
+  if (certain === true) {
+    clauses.push("certain = TRUE");
+  } else if (certain === false) {
+    clauses.push("certain = FALSE");
+  }
+
+  // Include netlocs with both www/non-www variants
+  if (includeNetlocs) {
+    const variants = new Set<string>();
+    for (const n of includeNetlocs) {
+      variants.add(n);
+      if (n.startsWith("www.")) {
+        variants.add(n.slice(4));
+      } else {
+        variants.add("www." + n);
+      }
+    }
+    const inList = Array.from(variants)
+      .sort()
+      .map((v) => `'${v}'`)
+      .join(", ");
+    clauses.push(`netloc IN (${inList})`);
+  }
+
+  // Exclude netlocs with both www/non-www variants
+  if (excludeNetlocs) {
+    const variants = new Set<string>();
+    for (const n of excludeNetlocs) {
+      variants.add(n);
+      if (n.startsWith("www.")) {
+        variants.add(n.slice(4));
+      } else {
+        variants.add("www." + n);
+      }
+    }
+    const exList = Array.from(variants)
+      .sort()
+      .map((v) => `'${v}'`)
+      .join(", ");
+    clauses.push(`netloc NOT IN (${exList})`);
+  }
+
+  // Include companies
+  if (includeCompanies) {
+    const companyList = includeCompanies
+      .map((c) => `ARRAY_CONTAINS(companies, '${c}')`)
+      .join(" OR ");
+    clauses.push(`(companies IS NOT NULL AND (${companyList}))`);
+  }
+
+  // Exclude companies
+  if (excludeCompanies) {
+    const companyList = excludeCompanies
+      .map((c) => `ARRAY_CONTAINS(companies, '${c}')`)
+      .join(" OR ");
+    clauses.push(`(companies IS NULL OR NOT (${companyList}))`);
+  }
+
+  // Include docs
+  if (includeDocs) {
+    const docHashes = includeDocs.map((doc) => `'${doc}'`).join(", ");
+    clauses.push(`doc_hash IN (${docHashes})`);
+  }
+
+  // Exclude docs
+  if (excludeDocs) {
+    const docHashes = excludeDocs.map((doc) => `'${doc}'`).join(", ");
+    clauses.push(`doc_hash NOT IN (${docHashes})`);
+  }
+
+  // Join everything
+  if (clauses.length > 0) {
+    sql.push("WHERE " + clauses.join(" AND "));
+  }
+
+  if (sql.length === 1) {
+    return null;
+  }
+
+  const sqlFilter = sql.join(" ");
+
+  // TODO: Validate SQL
+  return sqlFilter;
+}

package/src/test-utils/mocks.ts

@@ -0,0 +1,159 @@
+import {vi} from "bun:test";
+import type {
+  SearchResultSemantics,
+  SearchResponse,
+  SearchQuery,
+} from "../api/schemas";
+
+/**
+ * Shared mock objects and utilities for testing
+ * This file avoids circular dependencies by only importing types, not classes
+ */
+
+// Mock semantics data
+export const mockSemantics: SearchResultSemantics = {
+  origin_shard: 1,
+  chunks_total: 10,
+  chunks_matched: 5,
+  chunks_kept: 3,
+  similarity: 0.85,
+  bitstring: "101010",
+};
+
+// Mock search query
+export const mockSearchQuery: SearchQuery = {
+  instruction: "Search for relevant articles",
+  question: "artificial intelligence machine learning",
+  expansions: ["AI", "ML", "neural networks"],
+  brand_safety: null,
+  language: null,
+  continent: null,
+  region: null,
+  country: null,
+  sector: null,
+  industry_group: null,
+  industry: null,
+  sub_industry: null,
+  iab_tier_1: null,
+  iab_tier_2: null,
+  iab_tier_3: null,
+  iab_tier_4: null,
+  algorithm: "hybrid-2",
+  n_results: 10,
+  n_contextify: 256,
+  n_probes: 5,
+  min_similarity: 0.7,
+  pass_similarity: 0.5,
+  must_exclude: [],
+  must_include: [],
+  shard_selector: "default",
+  shard_reranker: "default",
+  search_lang: "en",
+  search_words: {},
+  search_intents: {},
+  companies: [],
+};
+
+// Mock search response
+export const mockSearchResponse: SearchResponse = {
+  url_hash: "abc123",
+  url: "https://example.com/article",
+  netloc: "example.com",
+  published: "2023-01-15",
+  visited: "2023-01-16",
+  language: "en",
+  author: "John Doe",
+  title: "Test Article Title",
+  description: "This is a test article description",
+  content:
+    "This is the full content of the test article. It contains multiple sentences and provides comprehensive information about the topic being discussed.",
+  best_chunk: "This is the most relevant chunk of content from the article.",
+  semantics: mockSemantics,
+};
+
+// Additional mock search responses for testing multiple results
+export const mockSearchResponse2: SearchResponse = {
+  url_hash: "def456",
+  url: "https://example.com/article2",
+  netloc: "example.com",
+  published: "2023-02-20",
+  visited: "2023-02-21",
+  language: "en",
+  author: "Jane Smith",
+  title: "Test Article 2",
+  description: "This is test article 2",
+  content:
+    "This is the full content of test article 2 about artificial intelligence.",
+  best_chunk: "Most relevant chunk from article 2.",
+  semantics: {
+    ...mockSemantics,
+    origin_shard: 2,
+    similarity: 0.92,
+  },
+};
+
+// Combined result (search query + search response)
+export const mockResult = {
+  ...mockSearchQuery,
+  ...mockSearchResponse,
+};
+
+/**
+ * Factory function to create a mock NosibleClient
+ * Returns an object that can be cast to NosibleClient type in tests
+ */
+export const createMockClient = () => ({
+  fastSearch: vi.fn(),
+  scrapeUrl: vi.fn(),
+  bulkSearch: vi.fn(),
+  aiSearch: vi.fn(),
+  fastSearches: vi.fn(),
+  topicTrend: vi.fn(),
+  llmClient: {
+    chat: {
+      completions: {
+        create: vi.fn(),
+      },
+    },
+  },
+  expansionsModel: "openai/gpt-4o",
+});
+
+/**
+ * Create mock search response variations
+ */
+export const createMockSearchResponse = (
+  overrides: Partial<SearchResponse>
+): SearchResponse => ({
+  ...mockSearchResponse,
+  ...overrides,
+});
+
+/**
+ * Create mock semantics variations
+ */
+export const createMockSemantics = (
+  overrides: Partial<SearchResultSemantics>
+): SearchResultSemantics => ({
+  ...mockSemantics,
+  ...overrides,
+});
+
+/**
+ * Create multiple mock search responses for testing result sets
+ */
+export const createMockSearchResponses = (count: number): SearchResponse[] => {
+  return Array.from({length: count}, (_, i) => ({
+    ...mockSearchResponse,
+    url_hash: `hash${i + 1}`,
+    url: `https://example.com/article${i + 1}`,
+    title: `Test Article ${i + 1}`,
+    description: `This is test article ${i + 1}`,
+    content: `This is the full content of test article ${i + 1}.`,
+    semantics: {
+      ...mockSemantics,
+      origin_shard: i + 1,
+      similarity: 0.85 - i * 0.05,
+    },
+  }));
+};

package/src/topicTrend/topicTrend.ts

@@ -0,0 +1,53 @@
+export interface TopicTrendRes {
+  message: string;
+  query: Query;
+  response: {[key: string]: number};
+}
+
+export interface Query {
+  query: string;
+  brand_safety: null;
+  language: null;
+  continent: null;
+  region: null;
+  country: null;
+  sector: null;
+  industry_group: null;
+  industry: null;
+  sub_industry: null;
+  iab_tier_1: null;
+  iab_tier_2: null;
+  iab_tier_3: null;
+  iab_tier_4: null;
+  n_probes: number;
+  shard_selector: string;
+  shard_reranker: string;
+  search_lang: string;
+  search_words: SearchWords;
+}
+
+export interface SearchWords {
+  [key: string]: number;
+}
+
+// TODO: How to handle when API return 500 error
+
+// TODO: Add logic to allow user to pass in start and end date and then remove these from the response
+export class TopicTrend {
+  data: TopicTrendRes;
+
+  constructor(data: TopicTrendRes) {
+    this.data = data;
+  }
+
+  peakDate(): Date {
+    const maxValue = Math.max(...Object.values(this.data.response));
+    const date = Object.keys(this.data.response).find(
+      (key) => this.data.response[key] === maxValue || undefined
+    );
+    if (!date) {
+      throw new Error("No peak date found");
+    }
+    return new Date(date);
+  }
+}

package/src/utils/browser.test.ts

@@ -0,0 +1,209 @@
+import {describe, test, expect, beforeEach, afterEach, mock} from "bun:test";
+import {downloadBlob} from "./browser";
+
+describe("downloadBlob", () => {
+  // Store original globals
+  let originalBlob: typeof Blob;
+  let originalURL: typeof URL;
+  let originalDocument: Document;
+
+  // Mock objects
+  let mockAnchor: any;
+  let mockBlob: any;
+  let mockObjectURL: string;
+  let createElementSpy: any;
+  let appendChildSpy: any;
+  let removeChildSpy: any;
+  let clickSpy: any;
+  let createObjectURLSpy: any;
+  let revokeObjectURLSpy: any;
+
+  beforeEach(() => {
+    // Save originals
+    originalBlob = globalThis.Blob;
+    originalURL = globalThis.URL;
+    originalDocument = globalThis.document as any;
+
+    // Setup mock anchor element
+    clickSpy = mock(() => {});
+    mockAnchor = {
+      href: "",
+      download: "",
+      click: clickSpy,
+    };
+
+    // Setup mock document
+    createElementSpy = mock(() => mockAnchor);
+    appendChildSpy = mock(() => {});
+    removeChildSpy = mock(() => {});
+
+    (globalThis as any).document = {
+      createElement: createElementSpy,
+      body: {
+        appendChild: appendChildSpy,
+        removeChild: removeChildSpy,
+      },
+    };
+
+    // Setup mock Blob
+    mockBlob = {type: ""};
+    (globalThis as any).Blob = mock((content: any[], options: any) => {
+      mockBlob.type = options.type;
+      return mockBlob;
+    });
+
+    // Setup mock URL
+    mockObjectURL = "blob:mock-url-12345";
+    createObjectURLSpy = mock(() => mockObjectURL);
+    revokeObjectURLSpy = mock(() => {});
+
+    (globalThis as any).URL = {
+      createObjectURL: createObjectURLSpy,
+      revokeObjectURL: revokeObjectURLSpy,
+    };
+  });
+
+  afterEach(() => {
+    // Restore originals
+    (globalThis as any).Blob = originalBlob;
+    (globalThis as any).URL = originalURL;
+    (globalThis as any).document = originalDocument;
+  });
+
+  test("should create a blob with correct data and MIME type", () => {
+    const testData = '{"name": "test"}';
+    const mimeType = "application/json";
+    const fileName = "test.json";
+
+    downloadBlob(testData, mimeType, fileName);
+
+    expect(globalThis.Blob).toHaveBeenCalledWith([testData], {type: mimeType});
+  });
+
+  test("should create an anchor element", () => {
+    downloadBlob("test data", "text/plain", "test.txt");
+
+    expect(createElementSpy).toHaveBeenCalledWith("a");
+  });
+
+  test("should set correct href and download attributes on anchor", () => {
+    const fileName = "data.csv";
+    const mimeType = "text/csv";
+    const data = "col1,col2\nval1,val2";
+
+    downloadBlob(data, mimeType, fileName);
+
+    expect(mockAnchor.href).toBe(mockObjectURL);
+    expect(mockAnchor.download).toBe(fileName);
+  });
+
+  test("should append anchor to body, click it, then remove it", () => {
+    downloadBlob("test", "text/plain", "test.txt");
+
+    expect(appendChildSpy).toHaveBeenCalledWith(mockAnchor);
+    expect(clickSpy).toHaveBeenCalled();
+    expect(removeChildSpy).toHaveBeenCalledWith(mockAnchor);
+  });
+
+  test("should create and revoke object URL", () => {
+    downloadBlob("test", "text/plain", "test.txt");
+
+    expect(createObjectURLSpy).toHaveBeenCalledWith(mockBlob);
+    expect(revokeObjectURLSpy).toHaveBeenCalledWith(mockObjectURL);
+  });
+
+  test("should handle JSON data correctly", () => {
+    const jsonData = JSON.stringify({users: [{id: 1, name: "Alice"}]});
+    const mimeType = "application/json";
+    const fileName = "users.json";
+
+    downloadBlob(jsonData, mimeType, fileName);
+
+    expect(globalThis.Blob).toHaveBeenCalledWith([jsonData], {type: mimeType});
+    expect(mockAnchor.download).toBe(fileName);
+  });
+
+  test("should handle CSV data correctly", () => {
+    const csvData = "name,age\nJohn,30\nJane,25";
+    const mimeType = "text/csv";
+    const fileName = "data.csv";
+
+    downloadBlob(csvData, mimeType, fileName);
+
+    expect(globalThis.Blob).toHaveBeenCalledWith([csvData], {type: mimeType});
+    expect(mockAnchor.download).toBe(fileName);
+  });
+
+  test("should handle empty data", () => {
+    const emptyData = "";
+    const mimeType = "text/plain";
+    const fileName = "empty.txt";
+
+    downloadBlob(emptyData, mimeType, fileName);
+
+    expect(globalThis.Blob).toHaveBeenCalledWith([emptyData], {type: mimeType});
+    expect(clickSpy).toHaveBeenCalled();
+  });
+
+  test("should handle special characters in filename", () => {
+    const fileName = "test file (1) [copy].txt";
+
+    downloadBlob("data", "text/plain", fileName);
+
+    expect(mockAnchor.download).toBe(fileName);
+  });
+
+  test("should execute operations in correct order", () => {
+    const callOrder: string[] = [];
+
+    const orderClickSpy = mock(() => {
+      callOrder.push("click");
+    });
+    const orderAnchor = {
+      href: "",
+      download: "",
+      click: orderClickSpy,
+    };
+
+    createElementSpy = mock(() => {
+      callOrder.push("createElement");
+      return orderAnchor;
+    });
+    appendChildSpy = mock(() => {
+      callOrder.push("appendChild");
+    });
+    removeChildSpy = mock(() => {
+      callOrder.push("removeChild");
+    });
+    createObjectURLSpy = mock(() => {
+      callOrder.push("createObjectURL");
+      return mockObjectURL;
+    });
+    revokeObjectURLSpy = mock(() => {
+      callOrder.push("revokeObjectURL");
+    });
+
+    (globalThis as any).document = {
+      createElement: createElementSpy,
+      body: {
+        appendChild: appendChildSpy,
+        removeChild: removeChildSpy,
+      },
+    };
+    (globalThis as any).URL = {
+      createObjectURL: createObjectURLSpy,
+      revokeObjectURL: revokeObjectURLSpy,
+    };
+
+    downloadBlob("test", "text/plain", "test.txt");
+
+    expect(callOrder).toEqual([
+      "createObjectURL",
+      "createElement",
+      "appendChild",
+      "click",
+      "removeChild",
+      "revokeObjectURL",
+    ]);
+  });
+});