nosible 0.2.14 → 0.2.18

package/dist/index.cjs

@@ -148,11 +148,11 @@ function generateSqlFilter(input) {
   }
   if (includeDocs) {
     const docHashes = includeDocs.map((doc) => `'${doc}'`).join(", ");
-    clauses.push(`doc_hash IN (${docHashes})`);
+    clauses.push(`docs IN (${docHashes})`);
   }
   if (excludeDocs) {
     const docHashes = excludeDocs.map((doc) => `'${doc}'`).join(", ");
-    clauses.push(`doc_hash NOT IN (${docHashes})`);
+    clauses.push(`docs NOT IN (${docHashes})`);
   }
   if (clauses.length > 0) {
     sql.push("WHERE " + clauses.join(" AND "));
@@ -386,6 +386,22 @@ var bulkResSchema = import_zod3.default.object({
   decrypt_using: import_zod3.default.string(),
   download_from: import_zod3.default.string()
 });
+var limitItemSchema = import_zod3.default.object({
+  name: import_zod3.default.string(),
+  query_type: import_zod3.default.string(),
+  duration_seconds: import_zod3.default.number(),
+  duration_string: import_zod3.default.string(),
+  limit: import_zod3.default.number(),
+  limited: import_zod3.default.boolean(),
+  remaining: import_zod3.default.number(),
+  period_start: import_zod3.default.string().nullable(),
+  period_end: import_zod3.default.string().nullable()
+});
+var limitsResponseSchema = import_zod3.default.object({
+  api_key_id: import_zod3.default.string(),
+  subscription_id: import_zod3.default.string(),
+  limits: import_zod3.default.array(limitItemSchema)
+});
 
 // src/utils/llm.ts
 var defaultModel = "google/gemini-2.0-flash-001";
@@ -576,21 +592,28 @@ var callNosibleApi = async ({
   baseUrl = "https://www.nosible.ai/search/v2",
   endpoint,
   body,
-  apiKey
+  apiKey,
+  method = "POST"
 }) => {
   const now = Date.now();
-  console.debug("Body: ", JSON.stringify(body, null, 2));
-  console.time(`Request-${endpoint}(${now})`);
+  if (method === "POST" && body && process.env.NOSIBLE_DEBUG) {
+    console.debug("Body: ", JSON.stringify(body, null, 2));
+  }
+  if (process.env.NOSIBLE_DEBUG) {
+    console.time(`Request-${endpoint}(${now})`);
+  }
   const response = await fetch(`${baseUrl}/${endpoint}`, {
-    method: "POST",
+    method,
     headers: {
       Accept: "application/json",
-      "Content-Type": "application/json",
+      ...method === "POST" ? { "Content-Type": "application/json" } : {},
       "Api-Key": `${apiKey}`
     },
-    body: JSON.stringify(body)
+    ...method === "POST" && body ? { body: JSON.stringify(body) } : {}
   });
-  console.timeEnd(`Request-${endpoint}(${now})`);
+  if (process.env.NOSIBLE_DEBUG) {
+    console.timeEnd(`Request-${endpoint}(${now})`);
+  }
   if (!response.ok) {
     console.error("Status: " + response.status);
     if (response.status === 401) {
@@ -974,175 +997,35 @@ class TopicTrend {
   }
 }
 
-// src/utils/userPlan.ts
+// src/utils/rateLimiters.ts
 var import_bottleneck = __toESM(require("bottleneck"));
-var planLimits = [
-  {
-    key: "basic",
-    name: "Legacy",
-    rateLimits: {
-      scrapeUrl: { minute: 60, month: 1400 },
-      bulk: { minute: 60, month: 1400 },
-      fast: { minute: 60, month: 14000 }
-    }
-  },
-  {
-    key: "pro",
-    name: "Legacy",
-    rateLimits: {
-      scrapeUrl: { minute: 60, month: 6700 },
-      bulk: { minute: 60, month: 6700 },
-      fast: { minute: 60, month: 67000 }
-    }
-  },
-  {
-    key: "pro+",
-    name: "Legacy",
-    rateLimits: {
-      scrapeUrl: { minute: 60, month: 32000 },
-      bulk: { minute: 60, month: 32000 },
-      fast: { minute: 60, month: 320000 }
-    }
-  },
-  {
-    key: "bus",
-    name: "Legacy",
-    rateLimits: {
-      scrapeUrl: { minute: 60, month: 200000 },
-      bulk: { minute: 60, month: 200000 },
-      fast: { minute: 60, month: 2000000 }
-    }
-  },
-  {
-    key: "bus+",
-    name: "Legacy",
-    rateLimits: {
-      scrapeUrl: { minute: 60, month: 500000 },
-      bulk: { minute: 60, month: 500000 },
-      fast: { minute: 120, month: 5000000 }
-    }
-  },
-  {
-    key: "prod",
-    name: "Legacy",
-    rateLimits: {
-      scrapeUrl: { minute: 60, month: 1500000 },
-      bulk: { minute: 60, month: 1500000 },
-      fast: { minute: 360, month: 15000000 }
-    }
-  },
-  {
-    key: "chat",
-    name: "Legacy",
-    rateLimits: {
-      scrapeUrl: { minute: 60, month: 1500000 },
-      bulk: { minute: 60, month: 1500000 },
-      fast: { minute: 360, month: 15000000 }
-    }
-  },
-  {
-    key: "self",
-    name: "Legacy",
-    rateLimits: {
-      scrapeUrl: { minute: 6000, month: 1500000 },
-      bulk: { minute: 6000, month: 1500000 },
-      fast: { minute: 36000, month: 15000000 }
-    }
-  },
-  {
-    key: "test",
-    name: "Free",
-    rateLimits: {
-      scrapeUrl: { minute: 60, month: 300 },
-      bulk: { minute: 60, month: 300 },
-      fast: { minute: 60, month: 3000 }
-    }
-  },
-  {
-    key: "cons",
-    name: "Consumer",
-    rateLimits: {
-      scrapeUrl: { minute: 60, month: 3000 },
-      bulk: { minute: 60, month: 3000 },
-      fast: { minute: 120, month: 30000 }
-    }
-  },
-  {
-    key: "stup",
-    name: "Startup",
-    rateLimits: {
-      scrapeUrl: { minute: 60, month: 30000 },
-      bulk: { minute: 60, month: 30000 },
-      fast: { minute: 360, month: 300000 }
-    }
-  },
-  {
-    key: "busn",
-    name: "Business",
-    rateLimits: {
-      scrapeUrl: { minute: 60, month: 300000 },
-      bulk: { minute: 60, month: 300000 },
-      fast: { minute: 360, month: 3000000 }
-    }
-  },
-  {
-    key: "ent",
-    name: "Enterprise",
-    rateLimits: {
-      scrapeUrl: { minute: 60, month: 1500000 },
-      bulk: { minute: 60, month: 1500000 },
-      fast: { minute: 360, month: 15000000 }
-    }
+var createChainedLimiter = (limitItems) => {
+  if (limitItems.length === 0) {
+    return new import_bottleneck.default;
   }
-];
-var getUserPlan = (apiKey) => {
-  if (apiKey.split("|").length > 2) {
-    throw new Error("Invalid API key");
-  }
-  const planKey = apiKey.split("|")[0];
-  const plan = planLimits.find((p) => p.key === planKey);
-  if (!plan) {
-    throw new Error("Invalid API key");
+  const sortedLimits = [...limitItems].sort((a, b) => a.duration_seconds - b.duration_seconds);
+  const limiters = sortedLimits.map((limitItem) => {
+    const intervalMs = limitItem.duration_seconds * 1000;
+    return new import_bottleneck.default({
+      reservoir: limitItem.remaining ?? limitItem.limit,
+      reservoirRefreshAmount: limitItem.limit,
+      reservoirRefreshInterval: intervalMs
+    });
+  });
+  let chainedLimiter = limiters[0];
+  for (let i = 1;i < limiters.length; i++) {
+    chainedLimiter = chainedLimiter.chain(limiters[i]);
   }
-  return plan;
+  return chainedLimiter;
 };
-var minMs = 60 * 1000;
-var monthMs = 60 * 60 * 24 * 30 * 1000;
-var getLimiters = (planKey) => {
-  const plan = getUserPlan(planKey);
-  const scrapeUrlMinLimiter = new import_bottleneck.default({
-    reservoir: plan.rateLimits.scrapeUrl.minute,
-    reservoirRefreshAmount: plan.rateLimits.scrapeUrl.minute,
-    reservoirRefreshInterval: minMs
-  });
-  const scrapeUrlMonthLimiter = new import_bottleneck.default({
-    reservoir: plan.rateLimits.scrapeUrl.month,
-    reservoirRefreshAmount: plan.rateLimits.scrapeUrl.month,
-    reservoirRefreshInterval: monthMs
-  });
-  const scrapeUrlLimiter = scrapeUrlMinLimiter.chain(scrapeUrlMonthLimiter);
-  const bulkMinLimiter = new import_bottleneck.default({
-    reservoir: plan.rateLimits.bulk.minute,
-    reservoirRefreshAmount: plan.rateLimits.bulk.minute,
-    reservoirRefreshInterval: minMs
-  });
-  const bulkMonthLimiter = new import_bottleneck.default({
-    reservoir: plan.rateLimits.bulk.month,
-    reservoirRefreshAmount: plan.rateLimits.bulk.month,
-    reservoirRefreshInterval: monthMs
-  });
-  const bulkLimiter = bulkMinLimiter.chain(bulkMonthLimiter);
-  const fastMinLimiter = new import_bottleneck.default({
-    reservoir: plan.rateLimits.fast.minute,
-    reservoirRefreshAmount: plan.rateLimits.fast.minute,
-    reservoirRefreshInterval: minMs
-  });
-  const fastMonthLimiter = new import_bottleneck.default({
-    reservoir: plan.rateLimits.fast.month,
-    reservoirRefreshAmount: plan.rateLimits.fast.month,
-    reservoirRefreshInterval: monthMs
-  });
-  const fastLimiter = fastMinLimiter.chain(fastMonthLimiter);
+var createLimiters = (limitsResponse) => {
+  const { limits } = limitsResponse;
+  const visitLimits = limits.filter((l) => l.query_type === "visit");
+  const slowLimits = limits.filter((l) => l.query_type === "slow");
+  const fastLimits = limits.filter((l) => l.query_type === "fast");
+  const scrapeUrlLimiter = createChainedLimiter(visitLimits);
+  const bulkLimiter = createChainedLimiter(slowLimits);
+  const fastLimiter = createChainedLimiter(fastLimits);
   return {
     scrapeUrl: scrapeUrlLimiter,
     bulk: bulkLimiter,
@@ -1708,23 +1591,26 @@ var nosibleClientParams = import_zod7.default.union([
     openAiBaseURL: import_zod7.default.string().optional(),
     sentimentModel: import_zod7.default.string().optional(),
     expansionsModel: import_zod7.default.string().optional(),
-    searchDefaults: clientDefaultSearchParamsSchema.optional()
+    searchDefaults: clientDefaultSearchParamsSchema.optional(),
+    rateLimit: import_zod7.default.boolean().optional()
   })
 ]);
 
 class NosibleClient {
   baseUrl = "https://www.nosible.ai/search/v2";
   apiKey;
+  api_key_version;
   llmClient;
   expansionsModel = "openai/gpt-4o";
   searchDefaults;
-  plan;
+  rateLimit = true;
   limiters;
   constructor(params) {
     let apiKeyToUse;
     let llmApiKeyToUse;
     let openAiBaseURL;
     let searchDefaultsToUse;
+    let rateLimitToUse = true;
     if (typeof params === "string") {
       apiKeyToUse = params;
     } else if (params) {
@@ -1732,6 +1618,7 @@ class NosibleClient {
       llmApiKeyToUse = params.llmApiKey || process.env.LLM_API_KEY;
       openAiBaseURL = params.openAiBaseURL;
       searchDefaultsToUse = params.searchDefaults;
+      rateLimitToUse = params.rateLimit ?? false;
     } else {
       apiKeyToUse = process.env.NOSIBLE_API_KEY;
       llmApiKeyToUse = process.env.LLM_API_KEY;
@@ -1749,8 +1636,7 @@ class NosibleClient {
     }
     this.apiKey = apiKeyToUse;
     this.searchDefaults = searchDefaultsToUse;
-    this.plan = getUserPlan(this.apiKey);
-    this.limiters = getLimiters(this.apiKey);
+    this.rateLimit = rateLimitToUse;
   }
   mergeWithDefaultSearch(params) {
     if (!this.searchDefaults) {
@@ -1765,43 +1651,70 @@ class NosibleClient {
     if (!apiKey) {
       throw new Error("API key is required");
     }
+    if (apiKey.startsWith("nos_sk_")) {
+      this.api_key_version = "v2";
+      this.apiKey = apiKey;
+      return;
+    }
     const splits = apiKey.split("|");
     if (splits.length !== 2) {
-      throw new Error("API key is invalid - expected format 'key_id|secret_key'");
+      throw new Error("API key is invalid - expected format 'plan_id|secret_key' (v1) or 'nos_sk_...' (v2)");
     }
+    this.api_key_version = "v1";
     this.apiKey = apiKey;
   }
-  async _request(endpoint, body) {
+  async _request(endpoint, body, method = "POST") {
     return callNosibleApi({
       endpoint,
       body,
-      apiKey: this.apiKey
+      apiKey: this.apiKey,
+      method
     });
   }
+  async _initializeLimiters() {
+    if (!this.rateLimit || this.limiters) {
+      return;
+    }
+    const limitsResponse = await this.limits();
+    this.limiters = createLimiters(limitsResponse);
+  }
+  async limits() {
+    const response = await this._request("limits", undefined, "GET");
+    return response;
+  }
   async fastSearch(params) {
-    let search;
-    const hasSearchInstance = (params2) => {
-      return "search" in params2;
-    };
-    if (hasSearchInstance(params)) {
-      if (params.search instanceof Search) {
-        search = params.search;
+    if (this.rateLimit && !this.limiters) {
+      await this._initializeLimiters();
+    }
+    const executeSearch = async () => {
+      let search;
+      const hasSearchInstance = (params2) => {
+        return "search" in params2;
+      };
+      if (hasSearchInstance(params)) {
+        if (params.search instanceof Search) {
+          search = params.search;
+        } else {
+          throw new Error("Invalid search parameter provided.");
+        }
       } else {
-        throw new Error("Invalid search parameter provided.");
+        const mergedParams = this.mergeWithDefaultSearch(params);
+        search = new Search(mergedParams);
       }
-    } else {
-      const mergedParams = this.mergeWithDefaultSearch(params);
-      search = new Search(mergedParams);
+      const body = await search.searchBody({ client: this });
+      const validatedBody = validateFastSearchParams(body);
+      const { message, query, response } = await this._request("fast-search", validatedBody);
+      const results = response.map((result) => {
+        const fullResult = { ...query, ...result };
+        return fullResult;
+      });
+      const resultSet = createResultSet(this, results);
+      return resultSet;
+    };
+    if (this.rateLimit && this.limiters) {
+      return this.limiters.fast.schedule(executeSearch);
     }
-    const body = await search.searchBody({ client: this });
-    const validatedBody = validateFastSearchParams(body);
-    const { message, query, response } = await this._request("fast-search", validatedBody);
-    const results = response.map((result) => {
-      const fullResult = { ...query, ...result };
-      return fullResult;
-    });
-    const resultSet = createResultSet(this, results);
-    return resultSet;
+    return executeSearch();
   }
   async fastSearches(params) {
     let searchSet;
@@ -1817,49 +1730,74 @@ class NosibleClient {
     return results;
   }
   async aiSearch(params) {
-    const json = await this._request("search", params);
-    const results = json.response.map((result) => {
-      const fullResult = { ...json.query, ...result };
-      return fullResult;
-    });
-    const resultSet = createResultSet(this, results);
-    return resultSet;
+    if (this.rateLimit && !this.limiters) {
+      await this._initializeLimiters();
+    }
+    const executeSearch = async () => {
+      const json = await this._request("search", params);
+      const results = json.response.map((result) => {
+        const fullResult = { ...json.query, ...result };
+        return fullResult;
+      });
+      const resultSet = createResultSet(this, results);
+      return resultSet;
+    };
+    if (this.rateLimit && this.limiters) {
+      return this.limiters.fast.schedule(executeSearch);
+    }
+    return executeSearch();
   }
   async bulkSearch(params) {
-    let search;
-    const hasSearchInstance = (params2) => {
-      return "search" in params2;
-    };
-    if (hasSearchInstance(params)) {
-      if (params.search instanceof Search) {
-        search = params.search;
+    if (this.rateLimit && !this.limiters) {
+      await this._initializeLimiters();
+    }
+    const executeSearch = async () => {
+      let search;
+      const hasSearchInstance = (params2) => {
+        return "search" in params2;
+      };
+      if (hasSearchInstance(params)) {
+        if (params.search instanceof Search) {
+          search = params.search;
+        } else {
+          throw new Error("Invalid search parameter provided.");
+        }
       } else {
-        throw new Error("Invalid search parameter provided.");
+        const mergedParams = this.mergeWithDefaultSearch(params);
+        search = new Search(mergedParams);
       }
-    } else {
-      const mergedParams = this.mergeWithDefaultSearch(params);
-      search = new Search(mergedParams);
+      const body = await search.searchBody({ client: this });
+      const validatedBody = validateBulkSearchParams(body);
+      const fileLocation = await this._request("bulk-search", validatedBody);
+      const json = await bulkSearchDownload({
+        downloadFrom: fileLocation.download_from,
+        decryptUsing: fileLocation.decrypt_using
+      });
+      const results = json.response.map((result) => {
+        const fullResult = { ...json.query, ...result };
+        return fullResult;
+      });
+      const resultSet = createResultSet(this, results);
+      return resultSet;
+    };
+    if (this.rateLimit && this.limiters) {
+      return this.limiters.bulk.schedule(executeSearch);
     }
-    const body = await search.searchBody({ client: this });
-    const validatedBody = validateBulkSearchParams(body);
-    const fileLocation = await this._request("bulk-search", validatedBody);
-    const json = await bulkSearchDownload({
-      downloadFrom: fileLocation.download_from,
-      decryptUsing: fileLocation.decrypt_using
-    });
-    const results = json.response.map((result) => {
-      const fullResult = { ...json.query, ...result };
-      return fullResult;
-    });
-    const resultSet = createResultSet(this, results);
-    return resultSet;
+    return executeSearch();
   }
   async scrapeUrl(url) {
-    return this.limiters.scrapeUrl.schedule(async () => {
+    if (this.rateLimit && !this.limiters) {
+      await this._initializeLimiters();
+    }
+    const executeScrape = async () => {
       const json = await this._request("scrape-url", { url });
       const webPage = new WebPageData(this, json.response);
       return webPage;
-    });
+    };
+    if (this.rateLimit && this.limiters) {
+      return this.limiters.scrapeUrl.schedule(executeScrape);
+    }
+    return executeScrape();
   }
   async topicTrend(query, sqlFilter) {
     const body = { query, sql_filter: sqlFilter };
@@ -1869,4 +1807,4 @@ class NosibleClient {
   }
 }
 
-//# debugId=A7861032A0192DD264756E2164756E21
+//# debugId=0207BA2B2FF9617B64756E2164756E21