nosible 0.2.13 → 0.2.16

package/README.md

@@ -30,7 +30,7 @@ bun add nosible
 
 **Requirements**:
 
-- Node.js 18+
+- Node.js 20+
 - TypeScript 5+ (for TypeScript projects)
 
 **Dependencies**:

package/dist/index.cjs

@@ -42,6 +42,10 @@ var __export = (target, all) => {
 // src/index.ts
 var exports_src = {};
 __export(exports_src, {
+  userSearchParamsSchema: () => userSearchParamsSchema,
+  snippetSchema: () => snippetSchema,
+  scrapeResponseSchema: () => scrapeResponseSchema,
+  scrapeFullResSchema: () => scrapeFullResSchema,
   WebPageData: () => WebPageData,
   TopicTrend: () => TopicTrend,
   SearchSet: () => SearchSet,
@@ -176,16 +180,16 @@ var snippetStatisticsSchema = import_zod2.z.object({
   images: import_zod2.z.number().optional()
 });
 var snippetSchema = import_zod2.z.object({
-  url_hash: import_zod2.z.string(),
-  snippet_hash: import_zod2.z.string(),
-  prev_snippet_hash: import_zod2.z.string().nullable(),
-  next_snippet_hash: import_zod2.z.string().nullable(),
-  content: import_zod2.z.string(),
-  words: import_zod2.z.string().optional(),
-  language: import_zod2.z.string(),
-  statistics: snippetStatisticsSchema.optional(),
-  links: import_zod2.z.record(import_zod2.z.string(), import_zod2.z.string()).optional(),
-  images: import_zod2.z.array(import_zod2.z.string()).optional()
+  url_hash: import_zod2.z.string().describe("Hash of the URL from which the snippet was extracted"),
+  snippet_hash: import_zod2.z.string().describe("A unique hash for the snippet"),
+  prev_snippet_hash: import_zod2.z.string().nullable().describe("Hash of the previous snippet in sequence"),
+  next_snippet_hash: import_zod2.z.string().nullable().describe("Hash of the next snippet in sequence"),
+  content: import_zod2.z.string().describe("The text content of the snippet"),
+  words: import_zod2.z.string().optional().describe("The words in the snippet"),
+  language: import_zod2.z.string().describe("The language of the snippet"),
+  statistics: snippetStatisticsSchema.optional().describe("Statistical information about the snippet"),
+  links: import_zod2.z.record(import_zod2.z.string(), import_zod2.z.string()).optional().describe("List of links associated with the snippet"),
+  images: import_zod2.z.array(import_zod2.z.string()).optional().describe("List of image URLs associated with the snippet")
 });
 var scrapeStatisticsSchema = import_zod2.z.object({
   snippets: import_zod2.z.number(),
@@ -252,20 +256,20 @@ var structuredSchema = import_zod2.z.object({
 });
 var urlTreeSchema = import_zod2.z.record(import_zod2.z.string(), import_zod2.z.any());
 var scrapeResponseSchema = import_zod2.z.object({
-  request: scrapeRequestSchema,
-  page: scrapePageSchema,
-  statistics: scrapeStatisticsSchema,
-  languages: import_zod2.z.record(import_zod2.z.string(), import_zod2.z.number()),
-  snippets: import_zod2.z.record(import_zod2.z.string(), snippetSchema),
-  full_text: import_zod2.z.string(),
-  metadata: metadataSchema,
-  structured: import_zod2.z.array(structuredSchema),
-  url_tree: urlTreeSchema
+  request: scrapeRequestSchema.describe("Details of the original scrape request including URL components and proxy settings"),
+  page: scrapePageSchema.describe("Basic page metadata like title, description, author, and dates"),
+  statistics: scrapeStatisticsSchema.describe("Content statistics including counts of snippets, words, characters, images, and other media"),
+  languages: import_zod2.z.record(import_zod2.z.string(), import_zod2.z.number()).describe("Language detection results with language codes and confidence scores"),
+  snippets: import_zod2.z.record(import_zod2.z.string(), snippetSchema).describe("Page content broken into manageable snippets with metadata"),
+  full_text: import_zod2.z.string().describe("Complete extracted text content from the page"),
+  metadata: metadataSchema.describe("Additional metadata extracted from the page as key-value pairs"),
+  structured: import_zod2.z.array(structuredSchema).describe("Structured data extracted from the page (JSON-LD, microdata, etc.)"),
+  url_tree: urlTreeSchema.describe("Hierarchical structure of URLs found on the page")
 });
 var scrapeFullResSchema = import_zod2.z.object({
-  message: import_zod2.z.string(),
-  added_to_batch: import_zod2.z.boolean(),
-  response: scrapeResponseSchema
+  message: import_zod2.z.string().describe("Status message describing the scrape operation result"),
+  added_to_batch: import_zod2.z.boolean().describe("Indicates if this scrape was added to a batch for processing"),
+  response: scrapeResponseSchema.describe("The complete scrape response containing all extracted data")
 });
 
 // src/api/schemas.ts
@@ -382,6 +386,22 @@ var bulkResSchema = import_zod3.default.object({
   decrypt_using: import_zod3.default.string(),
   download_from: import_zod3.default.string()
 });
+var limitItemSchema = import_zod3.default.object({
+  name: import_zod3.default.string(),
+  query_type: import_zod3.default.string(),
+  duration_seconds: import_zod3.default.number(),
+  duration_string: import_zod3.default.string(),
+  limit: import_zod3.default.number(),
+  limited: import_zod3.default.boolean(),
+  remaining: import_zod3.default.number(),
+  period_start: import_zod3.default.string().nullable(),
+  period_end: import_zod3.default.string().nullable()
+});
+var limitsResponseSchema = import_zod3.default.object({
+  api_key_id: import_zod3.default.string(),
+  subscription_id: import_zod3.default.string(),
+  limits: import_zod3.default.array(limitItemSchema)
+});
 
 // src/utils/llm.ts
 var defaultModel = "google/gemini-2.0-flash-001";
@@ -572,21 +592,28 @@ var callNosibleApi = async ({
   baseUrl = "https://www.nosible.ai/search/v2",
   endpoint,
   body,
-  apiKey
+  apiKey,
+  method = "POST"
 }) => {
   const now = Date.now();
-  console.debug("Body: ", JSON.stringify(body, null, 2));
-  console.time(`Request-${endpoint}(${now})`);
+  if (method === "POST" && body && process.env.NOSIBLE_DEBUG) {
+    console.debug("Body: ", JSON.stringify(body, null, 2));
+  }
+  if (process.env.NOSIBLE_DEBUG) {
+    console.time(`Request-${endpoint}(${now})`);
+  }
   const response = await fetch(`${baseUrl}/${endpoint}`, {
-    method: "POST",
+    method,
     headers: {
       Accept: "application/json",
-      "Content-Type": "application/json",
+      ...method === "POST" ? { "Content-Type": "application/json" } : {},
       "Api-Key": `${apiKey}`
     },
-    body: JSON.stringify(body)
+    ...method === "POST" && body ? { body: JSON.stringify(body) } : {}
   });
-  console.timeEnd(`Request-${endpoint}(${now})`);
+  if (process.env.NOSIBLE_DEBUG) {
+    console.timeEnd(`Request-${endpoint}(${now})`);
+  }
   if (!response.ok) {
     console.error("Status: " + response.status);
     if (response.status === 401) {
@@ -911,8 +938,11 @@ class WebPageData {
   }
   static async fromJson(client, inputPath) {
     const data = await importJson({ filePath: inputPath });
-    const validData = scrapeResponseSchema.parse(data);
-    return new WebPageData(client, validData);
+    const validData = scrapeResponseSchema.safeParse(data);
+    if (!validData.success) {
+      throw new Error("Provided JSON is not a valid scrape response");
+    }
+    return new WebPageData(client, validData.data);
   }
   async writeJson(outputPath) {
     return await exportJson(this.data, outputPath);
@@ -967,175 +997,35 @@ class TopicTrend {
   }
 }
 
-// src/utils/userPlan.ts
+// src/utils/rateLimiters.ts
 var import_bottleneck = __toESM(require("bottleneck"));
-var planLimits = [
-  {
-    key: "basic",
-    name: "Legacy",
-    rateLimits: {
-      scrapeUrl: { minute: 60, month: 1400 },
-      bulk: { minute: 60, month: 1400 },
-      fast: { minute: 60, month: 14000 }
-    }
-  },
-  {
-    key: "pro",
-    name: "Legacy",
-    rateLimits: {
-      scrapeUrl: { minute: 60, month: 6700 },
-      bulk: { minute: 60, month: 6700 },
-      fast: { minute: 60, month: 67000 }
-    }
-  },
-  {
-    key: "pro+",
-    name: "Legacy",
-    rateLimits: {
-      scrapeUrl: { minute: 60, month: 32000 },
-      bulk: { minute: 60, month: 32000 },
-      fast: { minute: 60, month: 320000 }
-    }
-  },
-  {
-    key: "bus",
-    name: "Legacy",
-    rateLimits: {
-      scrapeUrl: { minute: 60, month: 200000 },
-      bulk: { minute: 60, month: 200000 },
-      fast: { minute: 60, month: 2000000 }
-    }
-  },
-  {
-    key: "bus+",
-    name: "Legacy",
-    rateLimits: {
-      scrapeUrl: { minute: 60, month: 500000 },
-      bulk: { minute: 60, month: 500000 },
-      fast: { minute: 120, month: 5000000 }
-    }
-  },
-  {
-    key: "prod",
-    name: "Legacy",
-    rateLimits: {
-      scrapeUrl: { minute: 60, month: 1500000 },
-      bulk: { minute: 60, month: 1500000 },
-      fast: { minute: 360, month: 15000000 }
-    }
-  },
-  {
-    key: "chat",
-    name: "Legacy",
-    rateLimits: {
-      scrapeUrl: { minute: 60, month: 1500000 },
-      bulk: { minute: 60, month: 1500000 },
-      fast: { minute: 360, month: 15000000 }
-    }
-  },
-  {
-    key: "self",
-    name: "Legacy",
-    rateLimits: {
-      scrapeUrl: { minute: 6000, month: 1500000 },
-      bulk: { minute: 6000, month: 1500000 },
-      fast: { minute: 36000, month: 15000000 }
-    }
-  },
-  {
-    key: "test",
-    name: "Free",
-    rateLimits: {
-      scrapeUrl: { minute: 60, month: 300 },
-      bulk: { minute: 60, month: 300 },
-      fast: { minute: 60, month: 3000 }
-    }
-  },
-  {
-    key: "cons",
-    name: "Consumer",
-    rateLimits: {
-      scrapeUrl: { minute: 60, month: 3000 },
-      bulk: { minute: 60, month: 3000 },
-      fast: { minute: 120, month: 30000 }
-    }
-  },
-  {
-    key: "stup",
-    name: "Startup",
-    rateLimits: {
-      scrapeUrl: { minute: 60, month: 30000 },
-      bulk: { minute: 60, month: 30000 },
-      fast: { minute: 360, month: 300000 }
-    }
-  },
-  {
-    key: "busn",
-    name: "Business",
-    rateLimits: {
-      scrapeUrl: { minute: 60, month: 300000 },
-      bulk: { minute: 60, month: 300000 },
-      fast: { minute: 360, month: 3000000 }
-    }
-  },
-  {
-    key: "ent",
-    name: "Enterprise",
-    rateLimits: {
-      scrapeUrl: { minute: 60, month: 1500000 },
-      bulk: { minute: 60, month: 1500000 },
-      fast: { minute: 360, month: 15000000 }
-    }
-  }
-];
-var getUserPlan = (apiKey) => {
-  if (apiKey.split("|").length > 2) {
-    throw new Error("Invalid API key");
+var createChainedLimiter = (limitItems) => {
+  if (limitItems.length === 0) {
+    return new import_bottleneck.default;
   }
-  const planKey = apiKey.split("|")[0];
-  const plan = planLimits.find((p) => p.key === planKey);
-  if (!plan) {
-    throw new Error("Invalid API key");
+  const sortedLimits = [...limitItems].sort((a, b) => a.duration_seconds - b.duration_seconds);
+  const limiters = sortedLimits.map((limitItem) => {
+    const intervalMs = limitItem.duration_seconds * 1000;
+    return new import_bottleneck.default({
+      reservoir: limitItem.remaining ?? limitItem.limit,
+      reservoirRefreshAmount: limitItem.limit,
+      reservoirRefreshInterval: intervalMs
+    });
+  });
+  let chainedLimiter = limiters[0];
+  for (let i = 1;i < limiters.length; i++) {
+    chainedLimiter = chainedLimiter.chain(limiters[i]);
   }
-  return plan;
+  return chainedLimiter;
 };
-var minMs = 60 * 1000;
-var monthMs = 60 * 60 * 24 * 30 * 1000;
-var getLimiters = (planKey) => {
-  const plan = getUserPlan(planKey);
-  const scrapeUrlMinLimiter = new import_bottleneck.default({
-    reservoir: plan.rateLimits.scrapeUrl.minute,
-    reservoirRefreshAmount: plan.rateLimits.scrapeUrl.minute,
-    reservoirRefreshInterval: minMs
-  });
-  const scrapeUrlMonthLimiter = new import_bottleneck.default({
-    reservoir: plan.rateLimits.scrapeUrl.month,
-    reservoirRefreshAmount: plan.rateLimits.scrapeUrl.month,
-    reservoirRefreshInterval: monthMs
-  });
-  const scrapeUrlLimiter = scrapeUrlMinLimiter.chain(scrapeUrlMonthLimiter);
-  const bulkMinLimiter = new import_bottleneck.default({
-    reservoir: plan.rateLimits.bulk.minute,
-    reservoirRefreshAmount: plan.rateLimits.bulk.minute,
-    reservoirRefreshInterval: minMs
-  });
-  const bulkMonthLimiter = new import_bottleneck.default({
-    reservoir: plan.rateLimits.bulk.month,
-    reservoirRefreshAmount: plan.rateLimits.bulk.month,
-    reservoirRefreshInterval: monthMs
-  });
-  const bulkLimiter = bulkMinLimiter.chain(bulkMonthLimiter);
-  const fastMinLimiter = new import_bottleneck.default({
-    reservoir: plan.rateLimits.fast.minute,
-    reservoirRefreshAmount: plan.rateLimits.fast.minute,
-    reservoirRefreshInterval: minMs
-  });
-  const fastMonthLimiter = new import_bottleneck.default({
-    reservoir: plan.rateLimits.fast.month,
-    reservoirRefreshAmount: plan.rateLimits.fast.month,
-    reservoirRefreshInterval: monthMs
-  });
-  const fastLimiter = fastMinLimiter.chain(fastMonthLimiter);
+var createLimiters = (limitsResponse) => {
+  const { limits } = limitsResponse;
+  const visitLimits = limits.filter((l) => l.query_type === "visit");
+  const slowLimits = limits.filter((l) => l.query_type === "slow");
+  const fastLimits = limits.filter((l) => l.query_type === "fast");
+  const scrapeUrlLimiter = createChainedLimiter(visitLimits);
+  const bulkLimiter = createChainedLimiter(slowLimits);
+  const fastLimiter = createChainedLimiter(fastLimits);
   return {
     scrapeUrl: scrapeUrlLimiter,
     bulk: bulkLimiter,
@@ -1701,23 +1591,26 @@ var nosibleClientParams = import_zod7.default.union([
     openAiBaseURL: import_zod7.default.string().optional(),
     sentimentModel: import_zod7.default.string().optional(),
     expansionsModel: import_zod7.default.string().optional(),
-    searchDefaults: clientDefaultSearchParamsSchema.optional()
+    searchDefaults: clientDefaultSearchParamsSchema.optional(),
+    rateLimit: import_zod7.default.boolean().optional()
   })
 ]);
 
 class NosibleClient {
   baseUrl = "https://www.nosible.ai/search/v2";
   apiKey;
+  api_key_version;
   llmClient;
   expansionsModel = "openai/gpt-4o";
   searchDefaults;
-  plan;
+  rateLimit = true;
   limiters;
   constructor(params) {
     let apiKeyToUse;
     let llmApiKeyToUse;
     let openAiBaseURL;
     let searchDefaultsToUse;
+    let rateLimitToUse = true;
     if (typeof params === "string") {
       apiKeyToUse = params;
     } else if (params) {
@@ -1725,6 +1618,7 @@ class NosibleClient {
       llmApiKeyToUse = params.llmApiKey || process.env.LLM_API_KEY;
       openAiBaseURL = params.openAiBaseURL;
       searchDefaultsToUse = params.searchDefaults;
+      rateLimitToUse = params.rateLimit ?? false;
     } else {
       apiKeyToUse = process.env.NOSIBLE_API_KEY;
       llmApiKeyToUse = process.env.LLM_API_KEY;
@@ -1742,8 +1636,7 @@ class NosibleClient {
     }
     this.apiKey = apiKeyToUse;
     this.searchDefaults = searchDefaultsToUse;
-    this.plan = getUserPlan(this.apiKey);
-    this.limiters = getLimiters(this.apiKey);
+    this.rateLimit = rateLimitToUse;
   }
   mergeWithDefaultSearch(params) {
     if (!this.searchDefaults) {
@@ -1758,43 +1651,70 @@ class NosibleClient {
     if (!apiKey) {
       throw new Error("API key is required");
     }
+    if (apiKey.startsWith("nos_sk_")) {
+      this.api_key_version = "v2";
+      this.apiKey = apiKey;
+      return;
+    }
     const splits = apiKey.split("|");
     if (splits.length !== 2) {
-      throw new Error("API key is invalid - expected format 'key_id|secret_key'");
+      throw new Error("API key is invalid - expected format 'plan_id|secret_key' (v1) or 'nos_sk_...' (v2)");
     }
+    this.api_key_version = "v1";
     this.apiKey = apiKey;
   }
-  async _request(endpoint, body) {
+  async _request(endpoint, body, method = "POST") {
     return callNosibleApi({
       endpoint,
       body,
-      apiKey: this.apiKey
+      apiKey: this.apiKey,
+      method
     });
   }
+  async _initializeLimiters() {
+    if (!this.rateLimit || this.limiters) {
+      return;
+    }
+    const limitsResponse = await this.limits();
+    this.limiters = createLimiters(limitsResponse);
+  }
+  async limits() {
+    const response = await this._request("limits", undefined, "GET");
+    return response;
+  }
   async fastSearch(params) {
-    let search;
-    const hasSearchInstance = (params2) => {
-      return "search" in params2;
-    };
-    if (hasSearchInstance(params)) {
-      if (params.search instanceof Search) {
-        search = params.search;
+    if (this.rateLimit && !this.limiters) {
+      await this._initializeLimiters();
+    }
+    const executeSearch = async () => {
+      let search;
+      const hasSearchInstance = (params2) => {
+        return "search" in params2;
+      };
+      if (hasSearchInstance(params)) {
+        if (params.search instanceof Search) {
+          search = params.search;
+        } else {
+          throw new Error("Invalid search parameter provided.");
+        }
       } else {
-        throw new Error("Invalid search parameter provided.");
+        const mergedParams = this.mergeWithDefaultSearch(params);
+        search = new Search(mergedParams);
       }
-    } else {
-      const mergedParams = this.mergeWithDefaultSearch(params);
-      search = new Search(mergedParams);
+      const body = await search.searchBody({ client: this });
+      const validatedBody = validateFastSearchParams(body);
+      const { message, query, response } = await this._request("fast-search", validatedBody);
+      const results = response.map((result) => {
+        const fullResult = { ...query, ...result };
+        return fullResult;
+      });
+      const resultSet = createResultSet(this, results);
+      return resultSet;
+    };
+    if (this.rateLimit && this.limiters) {
+      return this.limiters.fast.schedule(executeSearch);
     }
-    const body = await search.searchBody({ client: this });
-    const validatedBody = validateFastSearchParams(body);
-    const { message, query, response } = await this._request("fast-search", validatedBody);
-    const results = response.map((result) => {
-      const fullResult = { ...query, ...result };
-      return fullResult;
-    });
-    const resultSet = createResultSet(this, results);
-    return resultSet;
+    return executeSearch();
   }
   async fastSearches(params) {
     let searchSet;
@@ -1810,49 +1730,74 @@ class NosibleClient {
     return results;
   }
   async aiSearch(params) {
-    const json = await this._request("search", params);
-    const results = json.response.map((result) => {
-      const fullResult = { ...json.query, ...result };
-      return fullResult;
-    });
-    const resultSet = createResultSet(this, results);
-    return resultSet;
+    if (this.rateLimit && !this.limiters) {
+      await this._initializeLimiters();
+    }
+    const executeSearch = async () => {
+      const json = await this._request("search", params);
+      const results = json.response.map((result) => {
+        const fullResult = { ...json.query, ...result };
+        return fullResult;
+      });
+      const resultSet = createResultSet(this, results);
+      return resultSet;
+    };
+    if (this.rateLimit && this.limiters) {
+      return this.limiters.fast.schedule(executeSearch);
+    }
+    return executeSearch();
   }
   async bulkSearch(params) {
-    let search;
-    const hasSearchInstance = (params2) => {
-      return "search" in params2;
-    };
-    if (hasSearchInstance(params)) {
-      if (params.search instanceof Search) {
-        search = params.search;
+    if (this.rateLimit && !this.limiters) {
+      await this._initializeLimiters();
+    }
+    const executeSearch = async () => {
+      let search;
+      const hasSearchInstance = (params2) => {
+        return "search" in params2;
+      };
+      if (hasSearchInstance(params)) {
+        if (params.search instanceof Search) {
+          search = params.search;
+        } else {
+          throw new Error("Invalid search parameter provided.");
+        }
       } else {
-        throw new Error("Invalid search parameter provided.");
+        const mergedParams = this.mergeWithDefaultSearch(params);
+        search = new Search(mergedParams);
       }
-    } else {
-      const mergedParams = this.mergeWithDefaultSearch(params);
-      search = new Search(mergedParams);
+      const body = await search.searchBody({ client: this });
+      const validatedBody = validateBulkSearchParams(body);
+      const fileLocation = await this._request("bulk-search", validatedBody);
+      const json = await bulkSearchDownload({
+        downloadFrom: fileLocation.download_from,
+        decryptUsing: fileLocation.decrypt_using
+      });
+      const results = json.response.map((result) => {
+        const fullResult = { ...json.query, ...result };
+        return fullResult;
+      });
+      const resultSet = createResultSet(this, results);
+      return resultSet;
+    };
+    if (this.rateLimit && this.limiters) {
+      return this.limiters.bulk.schedule(executeSearch);
     }
-    const body = await search.searchBody({ client: this });
-    const validatedBody = validateBulkSearchParams(body);
-    const fileLocation = await this._request("bulk-search", validatedBody);
-    const json = await bulkSearchDownload({
-      downloadFrom: fileLocation.download_from,
-      decryptUsing: fileLocation.decrypt_using
-    });
-    const results = json.response.map((result) => {
-      const fullResult = { ...json.query, ...result };
-      return fullResult;
-    });
-    const resultSet = createResultSet(this, results);
-    return resultSet;
+    return executeSearch();
   }
   async scrapeUrl(url) {
-    return this.limiters.scrapeUrl.schedule(async () => {
+    if (this.rateLimit && !this.limiters) {
+      await this._initializeLimiters();
+    }
+    const executeScrape = async () => {
       const json = await this._request("scrape-url", { url });
       const webPage = new WebPageData(this, json.response);
       return webPage;
-    });
+    };
+    if (this.rateLimit && this.limiters) {
+      return this.limiters.scrapeUrl.schedule(executeScrape);
+    }
+    return executeScrape();
   }
   async topicTrend(query, sqlFilter) {
     const body = { query, sql_filter: sqlFilter };
@@ -1862,4 +1807,4 @@ class NosibleClient {
   }
 }
 
-//# debugId=88064A0B7CBC67C664756E2164756E21
+//# debugId=71464A07E20B4B6E64756E2164756E21