nosible 0.1.5

package/src/api/api.test.ts

@@ -0,0 +1,366 @@
+import {describe, it, expect, beforeEach, afterEach, jest} from "bun:test";
+import {callNosibleApi} from ".";
+
+describe("callNosibleApi mock responses", () => {
+  const mockApiKey = "test-api-key";
+  const mockEndpoint = "test-endpoint";
+  const mockBody = {test: "data"};
+
+  // Store the original fetch function
+  const originalFetch = globalThis.fetch;
+
+  beforeEach(() => {
+    // Reset fetch mock before each test
+  });
+
+  afterEach(() => {
+    // Restore the original fetch function after each test
+    globalThis.fetch = originalFetch;
+  });
+
+  it("should handle 200 OK response correctly", async () => {
+    // Mock a successful response
+    const mockResponse = Promise.resolve({
+      ok: true,
+      status: 200,
+      json: () =>
+        Promise.resolve({
+          data: "test response data",
+          message: "success",
+        }),
+    });
+
+    // Replace global fetch with our mock
+    globalThis.fetch = jest.fn(() => mockResponse) as any;
+
+    const result = await callNosibleApi({
+      endpoint: mockEndpoint,
+      body: mockBody,
+      apiKey: mockApiKey,
+    });
+
+    expect(result).toEqual({
+      data: "test response data",
+      message: "success",
+    });
+
+    // Verify fetch was called with correct parameters
+    expect(globalThis.fetch).toHaveBeenCalledWith(
+      `https://www.nosible.ai/search/v2/${mockEndpoint}`,
+      {
+        method: "POST",
+        headers: {
+          Accept: "application/json",
+          "Content-Type": "application/json",
+          "Api-Key": mockApiKey,
+        },
+        body: JSON.stringify(mockBody),
+      }
+    );
+  });
+
+  it("should handle 401 Unauthorized response correctly", async () => {
+    // Mock a 401 response
+    const mockResponse = Promise.resolve({
+      ok: false,
+      status: 401,
+      json: () => Promise.resolve({}),
+      text: () => Promise.resolve("Unauthorized"),
+    });
+
+    // Replace global fetch with our mock
+    globalThis.fetch = jest.fn(() => mockResponse) as any;
+
+    // Expect the function to throw an error
+    await expect(
+      callNosibleApi({
+        endpoint: mockEndpoint,
+        body: mockBody,
+        apiKey: mockApiKey,
+      })
+    ).rejects.toThrow("Your API key is not valid.");
+
+    // Verify fetch was called with correct parameters
+    expect(globalThis.fetch).toHaveBeenCalledWith(
+      `https://www.nosible.ai/search/v2/${mockEndpoint}`,
+      {
+        method: "POST",
+        headers: {
+          Accept: "application/json",
+          "Content-Type": "application/json",
+          "Api-Key": mockApiKey,
+        },
+        body: JSON.stringify(mockBody),
+      }
+    );
+  });
+
+  it("should handle 422 with string_too_short error correctly", async () => {
+    // Mock a 422 response with string_too_short error
+    const mockResponse = Promise.resolve({
+      ok: false,
+      status: 422,
+      headers: {
+        get: (name: string) => {
+          if (name.toLowerCase() === "content-type") return "application/json";
+          return "";
+        },
+      },
+      json: () =>
+        Promise.resolve([
+          {
+            type: "string_too_short",
+            message: "API key is too short",
+          },
+        ]),
+      text: () => Promise.resolve("Bad Request"),
+    });
+
+    // Replace global fetch with our mock
+    globalThis.fetch = jest.fn(() => mockResponse) as any;
+
+    // Expect the function to throw an error
+    await expect(
+      callNosibleApi({
+        endpoint: mockEndpoint,
+        body: mockBody,
+        apiKey: mockApiKey,
+      })
+    ).rejects.toThrow("Your API key is not valid: Too Short.");
+  });
+
+  it("should handle 422 with general bad request correctly", async () => {
+    // Mock a 422 response with general error
+    const mockResponse = Promise.resolve({
+      ok: false,
+      status: 422,
+      headers: {
+        get: (name: string) => {
+          if (name.toLowerCase() === "content-type") return "application/json";
+          return "";
+        },
+      },
+      json: () =>
+        Promise.resolve([
+          {
+            message: "Bad request",
+          },
+        ]),
+      text: () => Promise.resolve("Bad Request"),
+    });
+
+    // Replace global fetch with our mock
+    globalThis.fetch = jest.fn(() => mockResponse) as any;
+
+    // Expect the function to throw an error
+    await expect(
+      callNosibleApi({
+        endpoint: mockEndpoint,
+        body: mockBody,
+        apiKey: mockApiKey,
+      })
+    ).rejects.toThrow("You made a bad request.");
+  });
+
+  it("should handle 429 rate limit error correctly", async () => {
+    // Mock a 429 response
+    const mockResponse = Promise.resolve({
+      ok: false,
+      status: 429,
+      headers: {
+        get: () => "",
+      },
+      json: () => Promise.resolve({}),
+      text: () => Promise.resolve("Rate Limit Exceeded"),
+    });
+
+    // Replace global fetch with our mock
+    globalThis.fetch = jest.fn(() => mockResponse) as any;
+
+    // Expect the function to throw an error
+    await expect(
+      callNosibleApi({
+        endpoint: mockEndpoint,
+        body: mockBody,
+        apiKey: mockApiKey,
+      })
+    ).rejects.toThrow("You have hit your rate limit.");
+  });
+
+  it("should handle 409 concurrent searches error correctly", async () => {
+    // Mock a 409 response
+    const mockResponse = Promise.resolve({
+      ok: false,
+      status: 409,
+      headers: {
+        get: () => "",
+      },
+      json: () => Promise.resolve({}),
+      text: () => Promise.resolve("Too Many Requests"),
+    });
+
+    // Replace global fetch with our mock
+    globalThis.fetch = jest.fn(() => mockResponse) as any;
+
+    // Expect the function to throw an error
+    await expect(
+      callNosibleApi({
+        endpoint: mockEndpoint,
+        body: mockBody,
+        apiKey: mockApiKey,
+      })
+    ).rejects.toThrow("Too many concurrent searches.");
+  });
+
+  it("should handle 500 internal server error correctly", async () => {
+    // Mock a 500 response
+    const mockResponse = Promise.resolve({
+      ok: false,
+      status: 500,
+      headers: {
+        get: () => "",
+      },
+      json: () => Promise.resolve({}),
+      text: () => Promise.resolve("Internal Server Error"),
+    });
+
+    // Replace global fetch with our mock
+    globalThis.fetch = jest.fn(() => mockResponse) as any;
+
+    // Expect the function to throw an error
+    await expect(
+      callNosibleApi({
+        endpoint: mockEndpoint,
+        body: mockBody,
+        apiKey: mockApiKey,
+      })
+    ).rejects.toThrow("An unexpected error occurred.");
+  });
+
+  it("should handle 502 bad gateway error correctly", async () => {
+    // Mock a 502 response
+    const mockResponse = Promise.resolve({
+      ok: false,
+      status: 502,
+      headers: {
+        get: () => "",
+      },
+      json: () => Promise.resolve({}),
+      text: () => Promise.resolve("Bad Gateway"),
+    });
+
+    // Replace global fetch with our mock
+    globalThis.fetch = jest.fn(() => mockResponse) as any;
+
+    // Expect the function to throw an error
+    await expect(
+      callNosibleApi({
+        endpoint: mockEndpoint,
+        body: mockBody,
+        apiKey: mockApiKey,
+      })
+    ).rejects.toThrow("NOSIBLE is currently restarting.");
+  });
+
+  it("should handle 504 gateway timeout error correctly", async () => {
+    // Mock a 504 response
+    const mockResponse = Promise.resolve({
+      ok: false,
+      status: 504,
+      headers: {
+        get: () => "",
+      },
+      json: () => Promise.resolve({}),
+      text: () => Promise.resolve("Gateway Timeout"),
+    });
+
+    // Replace global fetch with our mock
+    globalThis.fetch = jest.fn(() => mockResponse) as any;
+
+    // Expect the function to throw an error
+    await expect(
+      callNosibleApi({
+        endpoint: mockEndpoint,
+        body: mockBody,
+        apiKey: mockApiKey,
+      })
+    ).rejects.toThrow("NOSIBLE is currently overloaded.");
+  });
+
+  it("should handle fallback for other status codes", async () => {
+    // Mock a 418 response (I'm a teapot)
+    const mockResponse = Promise.resolve({
+      ok: false,
+      status: 418,
+      headers: {
+        get: () => "",
+      },
+      json: () => Promise.resolve({}),
+      text: () => Promise.resolve("I'm a teapot"),
+    });
+
+    // Replace global fetch with our mock
+    globalThis.fetch = jest.fn(() => mockResponse) as any;
+
+    // Expect the function to throw an error
+    await expect(
+      callNosibleApi({
+        endpoint: mockEndpoint,
+        body: mockBody,
+        apiKey: mockApiKey,
+      })
+    ).rejects.toThrow("HTTP error! status: 418");
+  });
+});
+
+describe("callNosibleApi", () => {
+  const apiKey = process.env.NOSIBLE_API_KEY;
+
+  // Only run the API test if we have a valid API key
+  const shouldRunApiTest = apiKey && apiKey.length > 0;
+
+  it("should respond with JSON when calling the real API", async () => {
+    if (!shouldRunApiTest) {
+      // Skip the test if no API key is available
+      expect(true).toBe(true);
+      console.log("Skipping API test - no valid API key found");
+      return;
+    }
+
+    // When we have a valid API key, test the actual API call
+    const response = await callNosibleApi({
+      endpoint: "fast-search",
+      body: {
+        question: "machine learning",
+        sql_filter: "SELECT loc FROM engine",
+        algorithm: "hybrid-2",
+        n_results: 10,
+        language: "en",
+        continent: "North America",
+        country: "United States",
+        sector: "Information Technology",
+      },
+      apiKey: apiKey,
+    });
+
+    // Verify that we get a response with the expected structure
+    expect(response).toBeDefined();
+    // The response should be an object
+    expect(typeof response).toBe("object");
+    // Log the response structure for debugging
+    console.log("Response keys:", Object.keys(response));
+    // It should have the expected properties
+    expect(response).toHaveProperty("message");
+    expect(response).toHaveProperty("query");
+    expect(response).toHaveProperty("response");
+    // Response should be an array
+    expect(Array.isArray(response.response)).toBe(true);
+    // Each item in response should have required properties
+    if (response.response.length > 0) {
+      const firstItem = response.response[0];
+      expect(firstItem).toHaveProperty("url");
+      expect(firstItem).toHaveProperty("title");
+      expect(firstItem).toHaveProperty("content");
+    }
+  });
+});

package/src/api/index.ts

@@ -0,0 +1,179 @@
+import {
+  algorithmEnum,
+  brandSafetyEnum,
+  continentEnum,
+  regionEnum,
+} from "./schemas";
+import {z} from "zod";
+
+/* ------------------ SEARCH ------------------ */
+
+// Number of results
+export const fastNResults = z.number().min(10).max(100).optional().default(100);
+export const searchNResults = z
+  .number()
+  .min(100)
+  .max(1000)
+  .optional()
+  .default(100);
+export const bulkNResults = z
+  .number()
+  .min(1000)
+  .max(10000)
+  .optional()
+  .default(1000);
+
+// AI search params for search only
+export const aiSearchParamsSchema = z.object({
+  prompt: z.string().min(25).max(2500),
+  agent: z.string().optional(),
+});
+export type SearchParamsType = z.infer<typeof aiSearchParamsSchema>;
+
+// Standard search params for fast and bulk search only
+export const stdSearchParams = z.object({
+  question: z.string(),
+  expansions: z.array(z.string()).optional(),
+  sql_filter: z.string().optional(),
+  algorithm: algorithmEnum.optional(),
+  n_results: z.number(),
+  n_probes: z.number().min(1).max(10).optional(),
+  n_contextify: z.number().min(64).max(1024).optional(),
+  min_similarity: z.number().min(0).max(1).optional(),
+  must_include: z.array(z.string()).max(100).optional(),
+  must_exclude: z.array(z.string()).max(100).optional(),
+  brand_safety: brandSafetyEnum.optional(),
+  language: z.string().min(2).max(2).optional(),
+  continent: continentEnum.optional(),
+  region: regionEnum.optional(),
+  country: z.string().optional(),
+  sector: z.string().optional(),
+  industry_group: z.string().optional(),
+  industry: z.string().optional(),
+  sub_industry: z.string().optional(),
+  iab_tier_1: z.string().optional(),
+  iab_tier_2: z.string().optional(),
+  iab_tier_3: z.string().optional(),
+  iab_tier_4: z.string().optional(),
+  companies: z.array(z.string()).max(30).optional(),
+});
+export type StdSearchParamsType = z.infer<typeof stdSearchParams>;
+
+// Fast search params
+const fastSearchParamsSchema = z.object({
+  ...stdSearchParams.shape,
+  n_results: fastNResults, // Overrides stdSearchParams
+});
+export type FastSearchParamsType = z.infer<typeof fastSearchParamsSchema>;
+
+// Bulk search params
+const bulkSearchParamsSchema = z.object({
+  ...stdSearchParams.shape,
+  n_results: bulkNResults, // Overrides stdSearchParams
+});
+type BulkSearchParamsType = z.infer<typeof bulkSearchParamsSchema>;
+
+// Validation Functions
+export const validateAiSearchParams = (params: StdSearchParamsType) => {
+  const validatedParams = aiSearchParamsSchema.parse(params);
+};
+
+export const validateFastSearchParams = (params: FastSearchParamsType) => {
+  const validatedParams = fastSearchParamsSchema.parse(params);
+  return validatedParams;
+};
+
+export const validateBulkSearchParams = (params: BulkSearchParamsType) => {
+  const validatedParams = bulkSearchParamsSchema.parse(params);
+  return validatedParams;
+};
+
+/* --------------- SCRAPE URL --------------- */
+const scapeUrlBody = z.object({
+  url: z.string(),
+});
+type ScrapeUrlBodyType = z.infer<typeof scapeUrlBody>;
+
+/* ------------------ TOPIC TREND ------------------ */
+const topicTrendBody = z.object({
+  query: z.string(),
+  sql_filter: z.string().optional(),
+});
+type TopicTrendBodyType = z.infer<typeof topicTrendBody>;
+
+/* ------------------ API CALL ------------------ */
+
+// Call API
+
+export const callNosibleApi = async ({
+  baseUrl = "https://www.nosible.ai/search/v2",
+  endpoint,
+  body,
+  apiKey,
+}: {
+  baseUrl?: string;
+  endpoint: string;
+  body: any;
+  apiKey: string;
+}): Promise<any> => {
+  /** Make authenticated POST request to the API */
+  const response = await fetch(`${baseUrl}/${endpoint}`, {
+    method: "POST",
+    headers: {
+      Accept: "application/json",
+      "Content-Type": "application/json",
+      "Api-Key": `${apiKey}`,
+    },
+    body: JSON.stringify(body),
+  });
+
+  /** Handle error responses with detailed logging */
+  if (!response.ok) {
+    console.error("Status: " + response.status);
+
+    if (response.status === 401) {
+      throw new Error("Your API key is not valid.");
+    }
+
+    if (response.status === 422) {
+      const contentType = response.headers.get("Content-Type") || "";
+      if (contentType.startsWith("application/json")) {
+        const body = await response.json();
+        const errorBody = Array.isArray(body) ? body[0] : body;
+        console.error(errorBody);
+        if (errorBody?.type === "string_too_short") {
+          throw new Error("Your API key is not valid: Too Short.");
+        }
+      }
+      throw new Error("You made a bad request.");
+    }
+
+    if (response.status === 429) {
+      throw new Error("You have hit your rate limit.");
+    }
+
+    if (response.status === 409) {
+      throw new Error("Too many concurrent searches.");
+    }
+
+    if (response.status === 500) {
+      throw new Error("An unexpected error occurred.");
+    }
+
+    if (response.status === 502) {
+      throw new Error("NOSIBLE is currently restarting.");
+    }
+
+    if (response.status === 504) {
+      throw new Error("NOSIBLE is currently overloaded.");
+    }
+
+    // Fallback for other status codes
+    console.error(await response.text());
+    throw new Error(`HTTP error! status: ${response.status}`);
+  } else {
+    /** Log successful response and return parsed data */
+    const data = await response.json();
+    return data;
+  }
+};

package/src/api/schemas.ts

@@ -0,0 +1,152 @@
+import z from "zod";
+
+export const queryTypeEnum = z.enum(["fast-search", "search", "bulk-search"]);
+type QueryType = z.infer<typeof queryTypeEnum>;
+
+export const algorithmEnum = z.enum([
+  "string",
+  "lexical",
+  "baseline",
+  "hamming",
+  "hybrid-1",
+  "hybrid-2",
+  "hybrid-3",
+  "company",
+]);
+export type AlgorithmEnum = z.infer<typeof algorithmEnum>;
+
+export const brandSafetyEnum = z.enum(["safe", "sensitive", "unsafe"]);
+export type BrandSafetyEnum = z.infer<typeof brandSafetyEnum>;
+
+export const continentEnum = z.enum([
+  "Africa",
+  "Asia",
+  "Europe",
+  "North America",
+  "Oceania",
+  "South America",
+  "Worldwide",
+]);
+export type ContinentEnum = z.infer<typeof continentEnum>;
+
+export const regionEnum = z.enum([
+  "Caribbean",
+  "Central Africa",
+  "Central America",
+  "Central Asia",
+  "Central Europe",
+  "East Africa",
+  "East Asia",
+  "Eastern Europe",
+  "Middle East",
+  "North Africa",
+  "North America",
+  "Northern Europe",
+  "Oceania",
+  "South America",
+  "South Asia",
+  "Southeast Asia",
+  "Southern Africa",
+  "Southern Europe",
+  "Sub-Saharan Africa",
+  "The Amazon Basin",
+  "The Andes",
+  "The Arctic Region",
+  "The Balkans",
+  "The Caucasus",
+  "The Horn of Africa",
+  "The Levant",
+  "The Middle East",
+  "The Sahel",
+  "West Africa",
+  "Western Europe",
+  "Worldwide",
+]);
+
+export type RegionEnum = z.infer<typeof regionEnum>;
+
+export interface SearchResType {
+  message: string;
+  query: SearchQuery;
+  response: SearchResponse[];
+}
+
+export const searchQuerySchema = z.object({
+  instruction: z.string(),
+  question: z.string(),
+  expansions: z.array(z.string()),
+  brand_safety: z.null(),
+  language: z.string().nullable(),
+  continent: z.string().nullable(),
+  region: z.string().nullable(),
+  country: z.string().nullable(),
+  sector: z.string().nullable(),
+  industry_group: z.string().nullable(),
+  industry: z.string().nullable(),
+  sub_industry: z.string().nullable(),
+  iab_tier_1: z.string().nullable(),
+  iab_tier_2: z.string().nullable(),
+  iab_tier_3: z.string().nullable(),
+  iab_tier_4: z.string().nullable(),
+  algorithm: z.string(),
+  n_results: z.number(),
+  n_contextify: z.number(),
+  n_probes: z.number(),
+  min_similarity: z.number(),
+  pass_similarity: z.number(),
+  must_exclude: z.array(z.string()),
+  must_include: z.array(z.string()),
+  shard_selector: z.string(),
+  shard_reranker: z.string(),
+  search_lang: z.string(),
+  search_words: z.record(z.string(), z.number()),
+  search_intents: z.record(z.string(), z.any()), // Empty object type
+  companies: z.array(z.any()),
+});
+
+export type SearchQuery = z.infer<typeof searchQuerySchema>;
+
+// SearchIntents is an empty object type
+export type SearchIntents = Record<string, any>;
+
+const searchResultSemanticsSchema = z.object({
+  origin_shard: z.number(),
+  chunks_total: z.number(),
+  chunks_matched: z.number(),
+  chunks_kept: z.number(),
+  similarity: z.number(),
+  bitstring: z.string(),
+});
+export type SearchResultSemantics = z.infer<typeof searchResultSemanticsSchema>;
+
+export const searchResponseSchema = z.object({
+  url_hash: z.string(),
+  url: z.string(),
+  netloc: z.string(),
+  published: z.string(),
+  visited: z.string(),
+  language: z.string(),
+  author: z.string(),
+  title: z.string(),
+  description: z.string(),
+  content: z.string(),
+  best_chunk: z.string(),
+  semantics: searchResultSemanticsSchema,
+});
+export type SearchResponse = z.infer<typeof searchResponseSchema>;
+
+// Bulk Search Response
+const bulkResSchema = z.object({
+  message: z.string(),
+  decrypt_using: z.string(),
+  download_from: z.string(),
+});
+export type BulkResType = z.infer<typeof bulkResSchema>;
+
+// Re-export scrape types from scrape.ts
+export {
+  scrapeResponseSchema,
+  scrapeFullResSchema,
+  type ScrapeResponse,
+  type ScrapeFullRes,
+} from "../scrape/types";