nosible 0.1.5

package/src/search/resultSet.io.test.ts

@@ -0,0 +1,320 @@
+import {describe, it, expect, beforeAll} from "bun:test";
+import {ResultSet} from "./resultSet";
+import {importCsv} from "../utils/file";
+import {randomUUID} from "node:crypto";
+
+/**
+ * Test configuration
+ */
+const TEST_DIR = `${process.cwd()}/tmp`; // Directory where test files will be created
+const CLEANUP = true; // Set to false to keep test files for inspection
+
+/**
+ * Helper function to conditionally cleanup test files
+ */
+const cleanupFile = async (filePath: string) => {
+  if (CLEANUP) {
+    const fs = await import("node:fs/promises");
+    await fs.unlink(filePath).catch(() => {});
+  } else {
+    console.log(`Keeping test file: ${filePath}`);
+  }
+};
+
+/**
+ * I/O tests for ResultSet with file system operations
+ * These tests require:
+ * - NOSIBLE_API_KEY environment variable to be set
+ * - Real network access to Nosible API
+ * - File system access for reading/writing
+ */
+describe("ResultSet - I/O Tests", () => {
+  let client: any;
+  let searchResultSet: ResultSet;
+
+  beforeAll(async () => {
+    try {
+      // Dynamic import to break circular dependency
+      const clientModule = await import("../client");
+      const {NosibleClient} = clientModule;
+      client = new NosibleClient();
+
+      // Create test directory if it doesn't exist
+      const fs = await import("node:fs/promises");
+      await fs.mkdir(TEST_DIR, {recursive: true}).catch(() => {});
+
+      // Get a real search result set
+      searchResultSet = await client.fastSearch({
+        question: "artificial intelligence",
+        nResults: 10,
+      });
+    } catch (error) {
+      console.warn("Skipping I/O tests due to setup error");
+      throw error;
+    }
+  });
+
+  describe("File export methods", () => {
+    it("should export ResultSet to JSON file", async () => {
+      const uniqueId = randomUUID();
+      const fileName = `${TEST_DIR}/test-results-${uniqueId}.json`;
+      await searchResultSet.writeToJson(fileName);
+
+      console.log(
+        `Successfully exported ${searchResultSet.results.length} results to JSON`
+      );
+
+      // Verify the file was created
+      const fs = await import("node:fs/promises");
+      await new Promise((resolve) => setTimeout(resolve, 100));
+
+      const fileExists = await fs
+        .access(fileName)
+        .then(() => true)
+        .catch(() => false);
+      expect(fileExists).toBe(true);
+
+      // Clean up
+      await cleanupFile(fileName);
+    });
+
+    it("should export ResultSet to NDJSON file", async () => {
+      const uniqueId = randomUUID();
+      const fileName = `${TEST_DIR}/test-results-${uniqueId}.ndjson`;
+      await searchResultSet.writeToNdjson(fileName);
+
+      console.log(
+        `Successfully exported ${searchResultSet.results.length} results to NDJSON`
+      );
+
+      // Verify the file was created
+      const fs = await import("node:fs/promises");
+      await new Promise((resolve) => setTimeout(resolve, 100));
+
+      const fileExists = await fs
+        .access(fileName)
+        .then(() => true)
+        .catch(() => false);
+      expect(fileExists).toBe(true);
+
+      // Clean up
+      await cleanupFile(fileName);
+    });
+
+    it("should export ResultSet to CSV file", async () => {
+      const uniqueId = randomUUID();
+      const fileName = `${TEST_DIR}/test-results-${uniqueId}.csv`;
+      await searchResultSet.writeToCsv(fileName);
+
+      console.log(
+        `Successfully exported ${searchResultSet.results.length} results to CSV`
+      );
+
+      // Verify the file was created
+      const fs = await import("node:fs/promises");
+      await new Promise((resolve) => setTimeout(resolve, 100));
+
+      const fileExists = await fs
+        .access(fileName)
+        .then(() => true)
+        .catch(() => false);
+      expect(fileExists).toBe(true);
+
+      // Read file and verify contents
+      const json = await importCsv({filePath: fileName});
+      expect(json).toBeDefined();
+      expect(json.length).toBeGreaterThan(0);
+      expect(json[1].origin_shard).toBeDefined();
+
+      // Clean up
+      await cleanupFile(fileName);
+    });
+
+    it("should export ResultSet to Parquet file", async () => {
+      const uniqueId = randomUUID();
+      const fileName = `${TEST_DIR}/test-results-${uniqueId}.parquet`;
+      await searchResultSet.writeToParquet(fileName);
+
+      console.log(
+        `Successfully exported ${searchResultSet.results.length} results to Parquet`
+      );
+
+      // Verify the file was created
+      const fs = await import("node:fs/promises");
+      await new Promise((resolve) => setTimeout(resolve, 100));
+
+      const fileExists = await fs
+        .access(fileName)
+        .then(() => true)
+        .catch(() => false);
+      expect(fileExists).toBe(true);
+
+      // Clean up
+      await cleanupFile(fileName);
+    });
+
+    it("should export ResultSet to IPC file", async () => {
+      const uniqueId = randomUUID();
+      const fileName = `${TEST_DIR}/test-results-${uniqueId}.ipc`;
+      await searchResultSet.writeToIpc(fileName);
+
+      console.log(
+        `Successfully exported ${searchResultSet.results.length} results to IPC`
+      );
+
+      // Verify the file was created
+      const fs = await import("node:fs/promises");
+      await new Promise((resolve) => setTimeout(resolve, 100));
+
+      const fileExists = await fs
+        .access(fileName)
+        .then(() => true)
+        .catch(() => false);
+      expect(fileExists).toBe(true);
+
+      // Clean up
+      await cleanupFile(fileName);
+    });
+  });
+
+  describe("Data conversion methods", () => {
+    it("should convert ResultSet to Polars DataFrame", async () => {
+      const df = await searchResultSet.toPolars();
+
+      expect(df).toBeDefined();
+      expect(df.height).toBeGreaterThan(0);
+      expect(df.width).toBeGreaterThan(0);
+
+      console.log(
+        `Successfully converted ${searchResultSet.results.length} results to Polars DataFrame`
+      );
+      console.log(`DataFrame shape: ${df.height} rows x ${df.width} columns`);
+    });
+  });
+
+  describe("Round-trip export/import - JSON", () => {
+    it("should verify exported JSON can be reimported", async () => {
+      const uniqueId = randomUUID();
+      const fileName = `${TEST_DIR}/test-roundtrip-${uniqueId}.json`;
+      await searchResultSet.writeToJson(fileName);
+
+      // Import the JSON file back
+      const reimportedResultSet = await ResultSet.fromFilePath(
+        fileName,
+        client
+      );
+
+      expect(reimportedResultSet).toBeDefined();
+      expect(reimportedResultSet.results.length).toBe(
+        searchResultSet.results.length
+      );
+
+      console.log(
+        `Successfully round-tripped ${reimportedResultSet.results.length} results through JSON`
+      );
+
+      // Clean up
+      await cleanupFile(fileName);
+    });
+
+    it("should produce identical JSON files after save -> load -> save cycle", async () => {
+      const fs = await import("node:fs/promises");
+      const uniqueId = randomUUID();
+      const file1 = `${TEST_DIR}/test-roundtrip-1-${uniqueId}.json`;
+      const file2 = `${TEST_DIR}/test-roundtrip-2-${uniqueId}.json`;
+
+      try {
+        // Save original results
+        await searchResultSet.writeToJson(file1);
+
+        // Load and verify
+        const reloadedResultSet = await ResultSet.fromFilePath(file1, client);
+        expect(reloadedResultSet.results.length).toBe(
+          searchResultSet.results.length
+        );
+
+        // Save again
+        await reloadedResultSet.writeToJson(file2);
+
+        // Compare the two JSON files
+        const json1Content = await fs.readFile(file1, "utf-8");
+        const json2Content = await fs.readFile(file2, "utf-8");
+
+        const json1 = JSON.parse(json1Content);
+        const json2 = JSON.parse(json2Content);
+
+        // Verify they have the same structure and content
+        expect(json1.length).toBe(json2.length);
+        expect(json1).toEqual(json2);
+
+        console.log(
+          `✓ Both JSON files are identical (${json1.length} results)`
+        );
+      } finally {
+        // Clean up both files
+        await cleanupFile(file1);
+        await cleanupFile(file2);
+      }
+    });
+  });
+
+  describe("Round-trip export/import - CSV", () => {
+    it("should verify exported CSV can be reimported", async () => {
+      const uniqueId = randomUUID();
+      const fileName = `${TEST_DIR}/test-roundtrip-${uniqueId}.csv`;
+      await searchResultSet.writeToCsv(fileName);
+
+      // Import the CSV file back
+      const reimportedResultSet = await ResultSet.fromFilePath(
+        fileName,
+        client
+      );
+
+      expect(reimportedResultSet).toBeDefined();
+      expect(reimportedResultSet.results.length).toBe(
+        searchResultSet.results.length
+      );
+
+      console.log(
+        `Successfully round-tripped ${reimportedResultSet.results.length} results through CSV`
+      );
+
+      // Clean up
+      await cleanupFile(fileName);
+    });
+
+    it("should produce identical CSV files after save -> load -> save cycle", async () => {
+      const fs = await import("node:fs/promises");
+      const uniqueId = randomUUID();
+      const file1 = `${TEST_DIR}/test-roundtrip-1-${uniqueId}.csv`;
+      const file2 = `${TEST_DIR}/test-roundtrip-2-${uniqueId}.csv`;
+
+      try {
+        // Save original results
+        await searchResultSet.writeToCsv(file1);
+
+        // Load and verify
+        const reloadedResultSet = await ResultSet.fromFilePath(file1, client);
+        expect(reloadedResultSet.results.length).toBe(
+          searchResultSet.results.length
+        );
+
+        // Save again
+        await reloadedResultSet.writeToCsv(file2);
+
+        // Compare the two CSV files
+        const csv1Content = await fs.readFile(file1, "utf-8");
+        const csv2Content = await fs.readFile(file2, "utf-8");
+
+        // Verify they have the same content
+        expect(csv1Content).toBe(csv2Content);
+
+        console.log(`✓ Both CSV files are identical`);
+      } finally {
+        // Clean up both files
+        await cleanupFile(file1);
+        await cleanupFile(file2);
+      }
+    });
+  });
+});

package/src/search/resultSet.test.ts

@@ -0,0 +1,368 @@
+import {describe, it, expect, beforeEach} from "bun:test";
+import {ResultSet} from "./resultSet";
+import {Result} from "./result";
+import {AnalyzeBy} from "./analyze";
+import type {SearchResponse} from "../api/schemas";
+import {
+  mockSemantics,
+  mockSearchResponse,
+  mockSearchQuery,
+  createMockClient,
+} from "../test-utils/mocks";
+
+// Create additional mock search responses
+const mockSearchResponse1: SearchResponse = {
+  ...mockSearchResponse,
+  url_hash: "abc123",
+  url: "https://example.com/article1",
+  title: "Test Article 1",
+  description: "This is test article 1",
+  content: "This is the full content of test article 1 about machine learning.",
+  best_chunk: "Most relevant chunk from article 1.",
+};
+
+const mockSearchResponse2: SearchResponse = {
+  ...mockSearchResponse,
+  url_hash: "def456",
+  url: "https://example.com/article2",
+  published: "2023-02-20",
+  visited: "2023-02-21",
+  author: "Jane Smith",
+  title: "Test Article 2",
+  description: "This is test article 2",
+  content:
+    "This is the full content of test article 2 about artificial intelligence.",
+  best_chunk: "Most relevant chunk from article 2.",
+  semantics: {
+    ...mockSemantics,
+    origin_shard: 2,
+    similarity: 0.92,
+  },
+};
+
+const mockClient = createMockClient() as any;
+
+// Create mock results
+let mockResult1: Result;
+let mockResult2: Result;
+
+describe("ResultSet - Logic Tests", () => {
+  beforeEach(() => {
+    // Initialize mock results
+    mockResult1 = new Result({
+      client: mockClient,
+      result: {...mockSearchQuery, ...mockSearchResponse1},
+    });
+
+    mockResult2 = new Result({
+      client: mockClient,
+      result: {...mockSearchQuery, ...mockSearchResponse2},
+    });
+  });
+
+  describe("constructor", () => {
+    it("should create a ResultSet instance with an array of results", () => {
+      const resultSet = new ResultSet([mockResult1, mockResult2]);
+
+      expect(resultSet).toBeInstanceOf(ResultSet);
+      expect(resultSet.results).toHaveLength(2);
+      expect(resultSet.results[0]).toBe(mockResult1);
+      expect(resultSet.results[1]).toBe(mockResult2);
+    });
+
+    it("should create a ResultSet instance with an empty array", () => {
+      const resultSet = new ResultSet([]);
+
+      expect(resultSet).toBeInstanceOf(ResultSet);
+      expect(resultSet.results).toHaveLength(0);
+    });
+  });
+
+  describe("fromResults", () => {
+    it("should create a ResultSet from SearchResponse array", () => {
+      const resultSet = ResultSet.fromResults(
+        [mockSearchResponse1, mockSearchResponse2],
+        mockSearchQuery,
+        mockClient
+      );
+
+      expect(resultSet).toBeInstanceOf(ResultSet);
+      expect(resultSet.results).toHaveLength(2);
+      expect(resultSet.results[0]).toBeInstanceOf(Result);
+      expect(resultSet.results[0]?.url_hash).toBe("abc123");
+      expect(resultSet.results[1]?.url_hash).toBe("def456");
+    });
+
+    it("should create a ResultSet from Result array", () => {
+      const resultSet = ResultSet.fromResults(
+        [mockResult1, mockResult2],
+        mockSearchQuery,
+        mockClient
+      );
+
+      expect(resultSet).toBeInstanceOf(ResultSet);
+      expect(resultSet.results).toHaveLength(2);
+      expect(resultSet.results[0]).toBe(mockResult1);
+      expect(resultSet.results[1]).toBe(mockResult2);
+    });
+
+    it("should create a ResultSet from mixed array of SearchResponse and Result", () => {
+      const resultSet = ResultSet.fromResults(
+        [mockSearchResponse1, mockResult2],
+        mockSearchQuery,
+        mockClient
+      );
+
+      expect(resultSet).toBeInstanceOf(ResultSet);
+      expect(resultSet.results).toHaveLength(2);
+      expect(resultSet.results[0]).toBeInstanceOf(Result);
+      expect(resultSet.results[1]).toBe(mockResult2);
+    });
+  });
+
+  describe("join", () => {
+    it("should join multiple ResultSets into one", () => {
+      const resultSet1 = new ResultSet([mockResult1]);
+      const resultSet2 = new ResultSet([mockResult2]);
+
+      const joinedResultSet = ResultSet.join([resultSet1, resultSet2]);
+
+      expect(joinedResultSet).toBeInstanceOf(ResultSet);
+      expect(joinedResultSet.results).toHaveLength(2);
+      expect(joinedResultSet.results[0]).toBe(mockResult1);
+      expect(joinedResultSet.results[1]).toBe(mockResult2);
+    });
+
+    it("should join three ResultSets", () => {
+      const mockResult3 = new Result({
+        client: mockClient,
+        result: {
+          ...mockSearchQuery,
+          ...mockSearchResponse1,
+          url_hash: "ghi789",
+        },
+      });
+
+      const resultSet1 = new ResultSet([mockResult1]);
+      const resultSet2 = new ResultSet([mockResult2]);
+      const resultSet3 = new ResultSet([mockResult3]);
+
+      const joinedResultSet = ResultSet.join([
+        resultSet1,
+        resultSet2,
+        resultSet3,
+      ]);
+
+      expect(joinedResultSet.results).toHaveLength(3);
+      expect(joinedResultSet.results[0]).toBe(mockResult1);
+      expect(joinedResultSet.results[1]).toBe(mockResult2);
+      expect(joinedResultSet.results[2]).toBe(mockResult3);
+    });
+
+    it("should handle empty ResultSets", () => {
+      const resultSet1 = new ResultSet([mockResult1]);
+      const emptyResultSet = new ResultSet([]);
+      const resultSet2 = new ResultSet([mockResult2]);
+
+      const joinedResultSet = ResultSet.join([
+        resultSet1,
+        emptyResultSet,
+        resultSet2,
+      ]);
+
+      expect(joinedResultSet.results).toHaveLength(2);
+      expect(joinedResultSet.results[0]).toBe(mockResult1);
+      expect(joinedResultSet.results[1]).toBe(mockResult2);
+    });
+
+    it("should handle all empty ResultSets", () => {
+      const emptyResultSet1 = new ResultSet([]);
+      const emptyResultSet2 = new ResultSet([]);
+
+      const joinedResultSet = ResultSet.join([
+        emptyResultSet1,
+        emptyResultSet2,
+      ]);
+
+      expect(joinedResultSet.results).toHaveLength(0);
+    });
+
+    it("should handle single ResultSet", () => {
+      const resultSet1 = new ResultSet([mockResult1, mockResult2]);
+
+      const joinedResultSet = ResultSet.join([resultSet1]);
+
+      expect(joinedResultSet.results).toHaveLength(2);
+      expect(joinedResultSet.results[0]).toBe(mockResult1);
+      expect(joinedResultSet.results[1]).toBe(mockResult2);
+    });
+
+    it("should handle empty array of ResultSets", () => {
+      const joinedResultSet = ResultSet.join([]);
+
+      expect(joinedResultSet).toBeInstanceOf(ResultSet);
+      expect(joinedResultSet.results).toHaveLength(0);
+    });
+
+    it("should preserve Result objects without creating copies", () => {
+      const resultSet1 = new ResultSet([mockResult1]);
+      const resultSet2 = new ResultSet([mockResult2]);
+
+      const joinedResultSet = ResultSet.join([resultSet1, resultSet2]);
+
+      // Should be the same object references, not copies
+      expect(joinedResultSet.results[0]).toBe(mockResult1);
+      expect(joinedResultSet.results[1]).toBe(mockResult2);
+    });
+
+    it("should handle ResultSets with multiple results each", () => {
+      const mockResult3 = new Result({
+        client: mockClient,
+        result: {
+          ...mockSearchQuery,
+          ...mockSearchResponse1,
+          url_hash: "ghi789",
+        },
+      });
+      const mockResult4 = new Result({
+        client: mockClient,
+        result: {
+          ...mockSearchQuery,
+          ...mockSearchResponse2,
+          url_hash: "jkl012",
+        },
+      });
+
+      const resultSet1 = new ResultSet([mockResult1, mockResult2]);
+      const resultSet2 = new ResultSet([mockResult3, mockResult4]);
+
+      const joinedResultSet = ResultSet.join([resultSet1, resultSet2]);
+
+      expect(joinedResultSet.results).toHaveLength(4);
+      expect(joinedResultSet.results[0]).toBe(mockResult1);
+      expect(joinedResultSet.results[1]).toBe(mockResult2);
+      expect(joinedResultSet.results[2]).toBe(mockResult3);
+      expect(joinedResultSet.results[3]).toBe(mockResult4);
+    });
+  });
+
+  describe("getResults", () => {
+    it("should return an array of result data objects", () => {
+      const resultSet = new ResultSet([mockResult1, mockResult2]);
+      const results = resultSet.getResults();
+
+      expect(results).toHaveLength(2);
+      expect(results[0]).toEqual(mockResult1.data());
+      expect(results[1]).toEqual(mockResult2.data());
+      expect(results[0]?.url_hash).toBe("abc123");
+      expect(results[1]?.url_hash).toBe("def456");
+    });
+
+    it("should return an empty array for empty ResultSet", () => {
+      const resultSet = new ResultSet([]);
+      const results = resultSet.getResults();
+
+      expect(results).toHaveLength(0);
+    });
+  });
+
+  describe("getFlattenResults", () => {
+    it("should return an array of flattened result objects", () => {
+      const resultSet = new ResultSet([mockResult1, mockResult2]);
+      const flattenedResults = resultSet.getFlattenResults();
+
+      expect(flattenedResults).toHaveLength(2);
+      expect(flattenedResults[0]).toEqual(mockResult1.flatten());
+      expect(flattenedResults[1]).toEqual(mockResult2.flatten());
+    });
+
+    it("should return an empty array for empty ResultSet", () => {
+      const resultSet = new ResultSet([]);
+      const flattenedResults = resultSet.getFlattenResults();
+
+      expect(flattenedResults).toHaveLength(0);
+    });
+  });
+
+  describe("findInResults", () => {
+    it("should find results matching the query using lunr", () => {
+      const resultSet = new ResultSet([mockResult1, mockResult2]);
+      const matches = resultSet.findInResults("machine learning");
+
+      expect(matches).toBeInstanceOf(Array);
+      // Should find the first result which contains "machine learning"
+      expect(matches.length).toBeGreaterThanOrEqual(1);
+    });
+
+    it("should return an empty array when no matches are found", () => {
+      const resultSet = new ResultSet([mockResult1, mockResult2]);
+      const matches = resultSet.findInResults("nonexistenttermsearchquery");
+
+      expect(matches).toBeInstanceOf(Array);
+      expect(matches).toHaveLength(0);
+    });
+
+    it("should return multiple results when query matches multiple documents", () => {
+      const resultSet = new ResultSet([mockResult1, mockResult2]);
+      const matches = resultSet.findInResults("article");
+
+      expect(matches).toBeInstanceOf(Array);
+      // Both results contain "article" in their content
+      expect(matches.length).toBeGreaterThanOrEqual(2);
+    });
+  });
+
+  describe("analyze", () => {
+    it("should call analyzeResults with correct parameters for categorical data", () => {
+      const resultSet = new ResultSet([mockResult1, mockResult2]);
+      const analyzeResult = resultSet.analyze(AnalyzeBy.netloc);
+
+      expect(analyzeResult).toBeDefined();
+      // Type assertion for categorical result
+      const categoricalResult = analyzeResult as Record<string, number>;
+      expect(categoricalResult["example.com"]).toBe(2);
+    });
+
+    it("should handle different AnalyzeBy types", () => {
+      const resultSet = new ResultSet([mockResult1, mockResult2]);
+
+      const byNetloc = resultSet.analyze(AnalyzeBy.netloc);
+      expect(byNetloc).toBeDefined();
+      const netlocResult = byNetloc as Record<string, number>;
+      expect(netlocResult["example.com"]).toBe(2);
+
+      const byAuthor = resultSet.analyze(AnalyzeBy.author);
+      expect(byAuthor).toBeDefined();
+
+      const bySimilarity = resultSet.analyze(AnalyzeBy.similarity);
+      expect(bySimilarity).toBeDefined();
+      expect("mean" in bySimilarity).toBe(true);
+    });
+  });
+
+  describe("integration scenarios", () => {
+    it("should handle a complete workflow: create, analyze, and search", () => {
+      const resultSet = ResultSet.fromResults(
+        [mockSearchResponse1, mockSearchResponse2],
+        mockSearchQuery,
+        mockClient
+      );
+
+      // Analyze
+      const analysis = resultSet.analyze(AnalyzeBy.netloc);
+      expect(analysis).toBeDefined();
+
+      // Search within results
+      const matches = resultSet.findInResults("article");
+      expect(matches.length).toBeGreaterThan(0);
+    });
+
+    it("should handle empty result sets gracefully", () => {
+      const emptyResultSet = new ResultSet([]);
+
+      expect(emptyResultSet.getResults()).toHaveLength(0);
+      expect(emptyResultSet.getFlattenResults()).toHaveLength(0);
+      expect(emptyResultSet.findInResults("query")).toHaveLength(0);
+    });
+  });
+});