nosible 0.1.5

package/src/index.ts

@@ -0,0 +1,4 @@
+import {NosibleClient} from "./client";
+import {Search} from "./search/search";
+
+export {NosibleClient, Search};

package/src/scrape/types.ts

@@ -0,0 +1,119 @@
+import {z} from "zod";
+
+const snippetStatisticsSchema = z.object({
+  sentences: z.number(),
+  words: z.number(),
+  characters: z.number(),
+  links: z.number().optional(),
+  images: z.number().optional(),
+});
+
+const snippetSchema = z.object({
+  url_hash: z.string(),
+  snippet_hash: z.string(),
+  prev_snippet_hash: z.string().nullable(),
+  next_snippet_hash: z.string().nullable(),
+  content: z.string(),
+  words: z.string().optional(),
+  language: z.string(),
+  statistics: snippetStatisticsSchema.optional(),
+  links: z.record(z.string(), z.string()).optional(),
+  images: z.array(z.string()).optional(),
+});
+export interface Snippet extends z.infer<typeof snippetSchema> {}
+
+const scrapeStatisticsSchema = z.object({
+  snippets: z.number(),
+  sentences: z.number(),
+  words: z.number(),
+  characters: z.number(),
+  images: z.number(),
+  videos: z.number(),
+  audio: z.number(),
+  tables: z.number(),
+  lists: z.number(),
+  blocks: z.number(),
+  links: z.number(),
+  files: z.number(),
+});
+export type ScrapeStatistics = z.infer<typeof scrapeStatisticsSchema>;
+
+const scrapePageSchema = z.object({
+  title: z.string(),
+  description: z.string(),
+  author: z.string(),
+  published: z.coerce.date(),
+  visited: z.coerce.date(),
+  certain: z.boolean().optional(),
+});
+export type ScrapePage = z.infer<typeof scrapePageSchema>;
+
+const querySchema = z.object({});
+
+const scrapeRequestSchema = z.object({
+  raw_url: z.string(),
+  url: z.string(),
+  hash: z.string(),
+  geo: z.string(),
+  proxy: z.string(),
+  scheme: z.string(),
+  netloc: z.string(),
+  prefix: z.string(),
+  domain: z.string(),
+  suffix: z.string(),
+  path: z.string(),
+  query: z.string(),
+  fragment: z.string(),
+  query_allowed: querySchema,
+  query_blocked: querySchema,
+});
+
+const metadataSchema = z.record(z.string(), z.any());
+
+const logoSchema = z.object({
+  "@type": z.string().optional(),
+  "@id": z.string().optional(),
+  caption: z.string().optional(),
+  inLanguage: z.string().optional(),
+  url: z.string(),
+  width: z.union([z.string(), z.number()]).optional(),
+  height: z.union([z.string(), z.number()]).optional(),
+});
+
+const publisherSchema = z.object({
+  "@type": z.string(),
+  name: z.string().optional(),
+  url: z.string().optional(),
+  logo: z.union([logoSchema, z.string()]).optional(),
+  sameAs: z.union([z.array(z.string()), z.string()]).optional(),
+});
+const structuredSchema = z.object({
+  "@context": z.string().optional(),
+  "@type": z.string().optional(),
+  headline: z.string().optional(),
+  publisher: publisherSchema.optional(),
+});
+
+const urlTreeSchema = z.record(z.string(), z.any());
+
+export const scrapeResponseSchema = z.object({
+  request: scrapeRequestSchema,
+  page: scrapePageSchema,
+  statistics: scrapeStatisticsSchema,
+  languages: z.record(z.string(), z.number()),
+  snippets: z.record(z.string(), snippetSchema),
+  full_text: z.string(),
+  metadata: metadataSchema,
+  structured: z.array(structuredSchema),
+  url_tree: urlTreeSchema,
+});
+
+export const scrapeFullResSchema = z.object({
+  message: z.string(),
+  added_to_batch: z.boolean(),
+  response: scrapeResponseSchema,
+});
+
+export type ScrapeFullRes = z.infer<typeof scrapeFullResSchema>;
+
+export type ScrapeResponse = z.infer<typeof scrapeResponseSchema>;

package/src/scrape/webPageData.test.ts

@@ -0,0 +1,302 @@
+import {describe, test, expect, afterEach} from "bun:test";
+import {mapSnippetToArray, WebPageData} from "./webPageData";
+import type {ScrapeResponse, Snippet} from "./types";
+import type {NosibleClient} from "../client";
+import {readFile, unlink} from "node:fs/promises";
+
+// Helper function to create a mock snippet
+export const createMockSnippet = (
+  snippet_hash: string,
+  prev_snippet_hash: string | null,
+  next_snippet_hash: string | null,
+  content: string = "Test content"
+): Snippet => ({
+  url_hash: "test_url_hash",
+  snippet_hash,
+  prev_snippet_hash,
+  next_snippet_hash,
+  content,
+  language: "en",
+});
+
+// Helper function to create a minimal mock ScrapeResponse
+export const createMockScrapeResponse = (
+  snippets: Record<string, Snippet>
+): ScrapeResponse => ({
+  request: {
+    raw_url: "https://example.com",
+    url: "https://example.com",
+    hash: "test_hash",
+    geo: "US",
+    proxy: "none",
+    scheme: "https",
+    netloc: "example.com",
+    prefix: "www",
+    domain: "example",
+    suffix: "com",
+    path: "/",
+    query: "",
+    fragment: "",
+    query_allowed: {},
+    query_blocked: {},
+  },
+  page: {
+    title: "Test Page",
+    description: "Test Description",
+    author: "Test Author",
+    published: new Date("2024-01-01"),
+    visited: new Date("2024-01-02"),
+  },
+  statistics: {
+    snippets: Object.keys(snippets).length,
+    sentences: 10,
+    words: 100,
+    characters: 500,
+    images: 0,
+    videos: 0,
+    audio: 0,
+    tables: 0,
+    lists: 0,
+    blocks: 5,
+    links: 3,
+    files: 0,
+  },
+  languages: {en: 1.0},
+  snippets,
+  full_text: "Test full text",
+  metadata: {},
+  structured: [],
+  url_tree: {},
+});
+
+describe("mapSnippetToArray", () => {
+  test("should correctly order a simple chain of 3 snippets", () => {
+    const snippet1 = createMockSnippet("hash1", null, "hash2", "First snippet");
+    const snippet2 = createMockSnippet(
+      "hash2",
+      "hash1",
+      "hash3",
+      "Second snippet"
+    );
+    const snippet3 = createMockSnippet("hash3", "hash2", null, "Third snippet");
+
+    const mockData = createMockScrapeResponse({
+      hash1: snippet1,
+      hash2: snippet2,
+      hash3: snippet3,
+    });
+
+    const result = mapSnippetToArray(mockData);
+
+    expect(result).toHaveLength(3);
+    expect(result[0]!.snippet_hash).toBe("hash1");
+    expect(result[1]!.snippet_hash).toBe("hash2");
+    expect(result[2]!.snippet_hash).toBe("hash3");
+    expect(result[0]!.content).toBe("First snippet");
+    expect(result[1]!.content).toBe("Second snippet");
+    expect(result[2]!.content).toBe("Third snippet");
+  });
+
+  test("should handle a single snippet", () => {
+    const snippet1 = createMockSnippet("hash1", null, null, "Only snippet");
+
+    const mockData = createMockScrapeResponse({
+      hash1: snippet1,
+    });
+
+    const result = mapSnippetToArray(mockData);
+
+    expect(result).toHaveLength(1);
+    expect(result[0]!.snippet_hash).toBe("hash1");
+    expect(result[0]!.content).toBe("Only snippet");
+  });
+
+  test("should correctly order snippets even when provided in random order", () => {
+    const snippet1 = createMockSnippet("hash1", null, "hash2");
+    const snippet2 = createMockSnippet("hash2", "hash1", "hash3");
+    const snippet3 = createMockSnippet("hash3", "hash2", "hash4");
+    const snippet4 = createMockSnippet("hash4", "hash3", null);
+
+    // Provide snippets in non-sequential order
+    const mockData = createMockScrapeResponse({
+      hash3: snippet3,
+      hash1: snippet1,
+      hash4: snippet4,
+      hash2: snippet2,
+    });
+
+    const result = mapSnippetToArray(mockData);
+
+    expect(result).toHaveLength(4);
+    expect(result[0]!.snippet_hash).toBe("hash1");
+    expect(result[1]!.snippet_hash).toBe("hash2");
+    expect(result[2]!.snippet_hash).toBe("hash3");
+    expect(result[3]!.snippet_hash).toBe("hash4");
+  });
+
+  test("should throw error when no first snippet is found", () => {
+    // All snippets have a prev_snippet_hash, so none is first
+    const snippet1 = createMockSnippet("hash1", "hash0", "hash2");
+    const snippet2 = createMockSnippet("hash2", "hash1", null);
+
+    const mockData = createMockScrapeResponse({
+      hash1: snippet1,
+      hash2: snippet2,
+    });
+
+    expect(() => mapSnippetToArray(mockData)).toThrow("No first snippet found");
+  });
+
+  test("should throw error when chain is broken (missing next snippet)", () => {
+    const snippet1 = createMockSnippet("hash1", null, "hash2");
+    const snippet3 = createMockSnippet("hash3", "hash2", null);
+    // hash2 is missing, but hash1 points to it
+
+    const mockData = createMockScrapeResponse({
+      hash1: snippet1,
+      hash3: snippet3,
+    });
+
+    expect(() => mapSnippetToArray(mockData)).toThrow("No next snippet found");
+  });
+
+  test("should handle long chain of snippets", () => {
+    const snippets: Record<string, Snippet> = {};
+    const count = 100;
+
+    for (let i = 0; i < count; i++) {
+      const hash = `hash${i}`;
+      const prevHash = i === 0 ? null : `hash${i - 1}`;
+      const nextHash = i === count - 1 ? null : `hash${i + 1}`;
+      snippets[hash] = createMockSnippet(
+        hash,
+        prevHash,
+        nextHash,
+        `Snippet ${i}`
+      );
+    }
+
+    const mockData = createMockScrapeResponse(snippets);
+    const result = mapSnippetToArray(mockData);
+
+    expect(result).toHaveLength(count);
+    for (let i = 0; i < count; i++) {
+      expect(result[i]!.snippet_hash).toBe(`hash${i}`);
+      expect(result[i]!.content).toBe(`Snippet ${i}`);
+    }
+  });
+
+  test("should maintain snippet properties during ordering", () => {
+    const snippet1 = createMockSnippet("hash1", null, "hash2");
+    snippet1.words = "test words";
+    snippet1.statistics = {
+      sentences: 5,
+      words: 50,
+      characters: 250,
+      links: 2,
+      images: 1,
+    };
+
+    const snippet2 = createMockSnippet("hash2", "hash1", null);
+
+    const mockData = createMockScrapeResponse({
+      hash1: snippet1,
+      hash2: snippet2,
+    });
+
+    const result = mapSnippetToArray(mockData);
+
+    expect(result[0]!.words).toBe("test words");
+    expect(result[0]!.statistics).toEqual({
+      sentences: 5,
+      words: 50,
+      characters: 250,
+      links: 2,
+      images: 1,
+    });
+  });
+
+  test("should return empty array when no snippets are provided", () => {
+    const mockData = createMockScrapeResponse({});
+
+    const result = mapSnippetToArray(mockData);
+
+    expect(result).toEqual([]);
+    expect(result).toHaveLength(0);
+  });
+});
+
+describe("WebPageData round trip test", () => {
+  const testFiles: string[] = [];
+
+  afterEach(async () => {
+    // Clean up test files
+    for (const file of testFiles) {
+      try {
+        await unlink(file);
+      } catch (error) {
+        // Ignore errors if file doesn't exist
+      }
+    }
+    testFiles.length = 0; // Clear the array
+  });
+
+  test("should serialize and deserialize WebPageData without data loss", async () => {
+    // Create a mock client
+    const mockClient = {} as NosibleClient;
+
+    // Create mock snippets
+    const snippet1 = createMockSnippet("hash1", null, "hash2", "First snippet");
+    const snippet2 = createMockSnippet(
+      "hash2",
+      "hash1",
+      "hash3",
+      "Second snippet"
+    );
+    const snippet3 = createMockSnippet("hash3", "hash2", null, "Third snippet");
+
+    // Create a comprehensive mock ScrapeResponse
+    const mockResponse = createMockScrapeResponse({
+      hash1: snippet1,
+      hash2: snippet2,
+      hash3: snippet3,
+    });
+
+    // Step 1: Create WebPageData from scrape response
+    const webPageData1 = new WebPageData(mockClient, mockResponse);
+
+    // Step 2: Write to JSON file (first time)
+    const jsonPath1 = "/tmp/test-webpage-data-1.json";
+    testFiles.push(jsonPath1);
+    await webPageData1.writeJson(jsonPath1);
+
+    // Step 3: Read JSON file and create new WebPageData
+    const webPageData2 = await WebPageData.fromJson(mockClient, jsonPath1);
+
+    // Step 4: Write to JSON file again (second time)
+    const jsonPath2 = "/tmp/test-webpage-data-2.json";
+    testFiles.push(jsonPath2);
+    await webPageData2.writeJson(jsonPath2);
+
+    // Step 5: Compare both JSON files
+    const json1Content = await readFile(jsonPath1, "utf-8");
+    const json2Content = await readFile(jsonPath2, "utf-8");
+
+    // Parse both JSON files to compare them properly
+    const json1Data = JSON.parse(json1Content);
+    const json2Data = JSON.parse(json2Content);
+
+    // The two JSON objects should be identical
+    expect(json2Data).toEqual(json1Data);
+
+    // Also verify that the WebPageData instances have the same data
+    expect(webPageData2.data).toEqual(webPageData1.data);
+    expect(webPageData2.page).toEqual(webPageData1.page);
+    expect(webPageData2.fullText).toEqual(webPageData1.fullText);
+    expect(webPageData2.metadata).toEqual(webPageData1.metadata);
+    expect(webPageData2.structured).toEqual(webPageData1.structured);
+    expect(webPageData2.urlTree).toEqual(webPageData1.urlTree);
+    expect(webPageData2.statistics).toEqual(webPageData1.statistics);
+    expect(webPageData2.getSnippets()).toEqual(webPageData1.getSnippets());
+  });
+});

package/src/scrape/webPageData.ts

@@ -0,0 +1,103 @@
+import z from "zod";
+import type {
+  ScrapePage,
+  ScrapeResponse,
+  ScrapeStatistics,
+  Snippet,
+} from "./types";
+import {scrapeResponseSchema} from "./types";
+import type {NosibleClient} from "../client";
+import {exportJson, importJson} from "../utils/file";
+
+export class WebPageData {
+  private client: NosibleClient;
+  public data: ScrapeResponse;
+  private snippets: Snippet[];
+  public page: ScrapePage;
+  public fullText: string;
+  public metadata: Record<string, any>;
+  public structured: any[];
+  public urlTree: Record<string, any>;
+  public statistics: ScrapeStatistics;
+  constructor(client: NosibleClient, res: ScrapeResponse) {
+    this.client = client;
+    this.data = res;
+    this.snippets = mapSnippetToArray(res);
+    this.page = this.data.page;
+    this.fullText = this.data.full_text;
+    this.metadata = this.data.metadata;
+    this.structured = this.data.structured;
+    this.urlTree = this.data.url_tree;
+    this.statistics = this.data.statistics;
+  }
+
+  static async fromJson(
+    client: NosibleClient,
+    inputPath: string
+  ): Promise<WebPageData> {
+    const data = await importJson({filePath: inputPath});
+    const validData = scrapeResponseSchema.parse(data) as ScrapeResponse;
+    return new WebPageData(client, validData);
+  }
+
+  public async writeJson(outputPath: string): Promise<void> {
+    return await exportJson(this.data, outputPath);
+  }
+
+  /*
+   * Returns an ordered array of snippets
+   * @returns {Snippet[]} An ordered array of snippets
+   */
+  public getSnippets(): Snippet[] {
+    return this.snippets;
+  }
+
+  /*
+   * Returns a snippet by hash key
+   * @param {string} hashKey - The hash key of the snippet
+   * @returns {Snippet | undefined} The snippet with the specified hash key
+   */
+  public getSnippet(hashKey: string): Snippet | undefined {
+    return this.snippets.find((snippet) => snippet.url_hash === hashKey);
+  }
+}
+
+export const mapSnippetToArray = (data: ScrapeResponse) => {
+  const snippets: Snippet[] = [];
+  const snippetKeys = Object.keys(data.snippets);
+
+  if (snippetKeys.length === 0) {
+    return snippets; // Return empty array if no snippets
+  }
+
+  // Unsorted snippets
+  const unsortedSnippets: Snippet[] = [];
+  snippetKeys.forEach((key) => {
+    unsortedSnippets.push(data.snippets[key] as Snippet);
+  });
+
+  // Find first snippet
+  const first = unsortedSnippets.find(
+    (snippet) => snippet.prev_snippet_hash === null
+  );
+  if (!first) {
+    throw new Error("No first snippet found");
+  }
+
+  // Add first snippet to array
+  snippets.push(first);
+
+  // Add remaining snippets to array
+  let current = first;
+  while (current.next_snippet_hash) {
+    const next = unsortedSnippets.find(
+      (snippet) => snippet.snippet_hash === current.next_snippet_hash
+    );
+    if (!next) {
+      throw new Error("No next snippet found");
+    }
+    snippets.push(next);
+    current = next;
+  }
+  return snippets;
+};