@supadata/js 1.0.2 → 1.0.4

package/README.md

@@ -16,7 +16,14 @@ npm install @supadata/js
 ## Usage
 
 ```typescript
-import { Supadata, Transcript } from '@supadata/js';
+import {
+  Supadata,
+  Transcript,
+  Scrape,
+  Map,
+  Crawl,
+  CrawlJob,
+} from '@supadata/js';
 
 // Initialize the client
 const supadata = new Supadata({
@@ -24,21 +31,30 @@ const supadata = new Supadata({
 });
 
 // Get YouTube transcript
-const transcript = await supadata.youtube.transcript({
-  videoId: 'VIDEO_ID',
+const transcript: Transcript = await supadata.youtube.transcript({
+  url: 'https://youtu.be/dQw4w9WgXcQ',
 });
 
 // Translate YouTube transcript
-const translated = await supadata.youtube.translate({
-  videoId: 'VIDEO_ID',
+const translated: Transcript = await supadata.youtube.translate({
+  videoId: 'dQw4w9WgXcQ',
   lang: 'es',
 });
 
 // Scrape web content
-const webContent = await supadata.web.scrape('https://supadata.ai');
+const webContent: Scrape = await supadata.web.scrape('https://supadata.ai');
 
 // Map website URLs
-const siteMap = await supadata.web.map('https://supadata.ai');
+const siteMap: Map = await supadata.web.map('https://supadata.ai');
+
+// Crawl website
+const crawl: Crawl = await supadata.web.crawl({
+  url: 'https://supadata.ai',
+  limit: 10,
+});
+
+// Get crawl job results
+const crawlResults: CrawlJob = await supadata.web.getCrawlResults(crawl.jobId);
 ```
 
 ## Error Handling
@@ -62,6 +78,10 @@ try {
 }
 ```
 
+## Example
+
+See the [example](./example) directory for a simple example of how to use the SDK.
+
 ## API Reference
 
 See the [Documentation](https://supadata.ai/documentation) for more details on all possible parameters and options.

package/dist/index.cjs

@@ -1,3 +1,3 @@
-'use strict';var e=class extends Error{error;details;documentationUrl;constructor(r){super(r.message||"An unexpected error occurred"),this.error=r.error||"internal-error",this.details=r.details||"An unexpected error occurred",this.documentationUrl=r.documentationUrl||"",this.name="SupadataError";}};var o={403:{error:"invalid-request",message:"Invalid or missing API key",details:"Please ensure you have provided a valid API key"},404:{error:"invalid-request",message:"Endpoint does not exist",details:"The API endpoint you are trying to access does not exist"},429:{error:"limit-exceeded",message:"Limit exceeded",details:"You have exceeded the allowed request rate or quota limits"}},g=(t,r)=>t in o?new e({...o[t],message:o[t].message,details:r||o[t].details}):new e({error:"internal-error",message:"An unexpected error occurred",details:r});var s=class{config;constructor(r){this.config=r;}async fetch(r,l){let m=`${this.config.baseUrl||"https://api.supadata.ai/v1"}${r.startsWith("/")?r:`/${r}`}`;if(l){let a=new URLSearchParams;Object.entries(l).forEach(([h,d])=>{d!=null&&a.append(h,String(d));}),m+=`?${a.toString()}`;}let n=await fetch(m,{method:"GET",headers:{"x-api-key":this.config.apiKey,"Content-Type":"application/json"}}),u=n.headers.get("content-type");if(!n.ok){if([403,404,429].includes(n.status)){let a=await n.json();throw g(n.status,a.message)}if(u?.includes("application/json")){let a=await n.json();throw new e(a)}else throw new e({error:"internal-error",message:"Unexpected error response format",details:await n.text()})}try{if(!u?.includes("application/json"))throw new e({error:"internal-error",message:"Invalid response format",details:"Expected JSON response but received different content type"});return await n.json()}catch(a){throw new e({error:"internal-error",message:"Failed to parse response",details:a instanceof Error?a.message:"Unknown error"})}}};var c=class extends s{async transcript(r){return this.fetch("/youtube/transcript",r)}async translate(r){return this.fetch("/youtube/transcript/translate",r)}};var p=class extends s{async scrape(r){return this.fetch("/web/scrape",{url:r})}async map(r){return this.fetch("/web/map",{url:r})}};var f=class{youtube;web;constructor(r){this.youtube=new c(r),this.web=new p(r);}};
-exports.BaseClient=s;exports.Supadata=f;exports.SupadataError=e;exports.WebService=p;exports.YouTubeService=c;//# sourceMappingURL=index.cjs.map
+'use strict';var a=class extends Error{error;details;documentationUrl;constructor(e){super(e.message||"An unexpected error occurred"),this.error=e.error||"internal-error",this.details=e.details||"An unexpected error occurred",this.documentationUrl=e.documentationUrl||"",this.name="SupadataError";}};var d={403:{error:"invalid-request",message:"Invalid or missing API key",details:"Please ensure you have provided a valid API key"},404:{error:"invalid-request",message:"Endpoint does not exist",details:"The API endpoint you are trying to access does not exist"},429:{error:"limit-exceeded",message:"Limit exceeded",details:"You have exceeded the allowed request rate or quota limits"}},f=(n,e)=>n in d?new a({...d[n],message:d[n].message,details:e||d[n].details}):new a({error:"internal-error",message:"An unexpected error occurred",details:e});var c=class{config;constructor(e){this.config=e;}async fetch(e,r={},s="GET"){let t=`${this.config.baseUrl||"https://api.supadata.ai/v1"}${e.startsWith("/")?e:`/${e}`}`;if(s==="GET"&&Object.keys(r).length>0){let p=new URLSearchParams;Object.entries(r).forEach(([o,m])=>{m!=null&&p.append(o,String(m));}),t+=`?${p.toString()}`;}return this.fetchUrl(t,s,r)}async fetchUrl(e,r="GET",s){let i={method:r,headers:{"x-api-key":this.config.apiKey,"Content-Type":"application/json"}};r==="POST"&&s&&(i.body=JSON.stringify(s));let t=await fetch(e,i),p=t.headers.get("content-type");if(!t.ok){if([403,404,429].includes(t.status)){let o=await t.json();throw f(t.status,o.message)}if(p?.includes("application/json")){let o=await t.json();throw new a(o)}else throw new a({error:"internal-error",message:"Unexpected error response format",details:await t.text()})}try{if(!p?.includes("application/json"))throw new a({error:"internal-error",message:"Invalid response format",details:"Expected JSON response but received different content type"});return await t.json()}catch(o){throw new a({error:"internal-error",message:"Failed to parse response",details:o instanceof Error?o.message:"Unknown error"})}}};var u=class extends c{async transcript(e){return this.fetch("/youtube/transcript",e)}async translate(e){return this.fetch("/youtube/transcript/translate",e)}};var g=class extends c{async scrape(e){return this.fetch("/web/scrape",{url:e})}async map(e){return this.fetch("/web/map",{url:e})}async crawl(e){return this.fetch("/web/crawl",e,"POST")}async getCrawlResults(e){let r,s=[],i;do r=await(i?this.fetchUrl(i):this.fetch(`/web/crawl/${e}`)),r.pages&&(s=[...s,...r.pages]),i=r.next;while(i);return r}};var T=class{youtube;web;constructor(e){this.youtube=new u(e),this.web=new g(e);}};
+exports.BaseClient=c;exports.Supadata=T;exports.SupadataError=a;exports.WebService=g;exports.YouTubeService=u;//# sourceMappingURL=index.cjs.map
 //# sourceMappingURL=index.cjs.map

package/dist/index.cjs.map

@@ -1 +1 @@
-{"version":3,"sources":["../src/types.ts","../src/gateway-error-mapper.ts","../src/client.ts","../src/services/youtube.ts","../src/services/web.ts","../src/index.ts"],"names":["SupadataError","error","GATEWAY_STATUS_ERRORS","mapGatewayError","statusCode","errorText","BaseClient","config","endpoint","params","url","queryParams","key","value","response","contentType","errorData","YouTubeService","WebService","Supadata"],"mappings":"aAqCaA,IAAAA,CAAAA,CAAN,cAA4B,KAAM,CACvC,MASA,OACA,CAAA,gBAAA,CAEA,YAAYC,CAKT,CAAA,CACD,MAAMA,CAAM,CAAA,OAAA,EAAW,8BAA8B,CACrD,CAAA,IAAA,CAAK,MAAQA,CAAM,CAAA,KAAA,EAAS,iBAC5B,IAAK,CAAA,OAAA,CAAUA,EAAM,OAAW,EAAA,8BAAA,CAChC,KAAK,gBAAmBA,CAAAA,CAAAA,CAAM,kBAAoB,EAClD,CAAA,IAAA,CAAK,KAAO,gBACd,CACF,ECvDA,IAAMC,CAAAA,CAAwB,CAC5B,GAAK,CAAA,CACH,MAAO,iBACP,CAAA,OAAA,CAAS,4BACT,CAAA,OAAA,CAAS,iDACX,CAAA,CACA,IAAK,CACH,KAAA,CAAO,kBACP,OAAS,CAAA,yBAAA,CACT,QAAS,0DACX,CAAA,CACA,IAAK,CACH,KAAA,CAAO,iBACP,OAAS,CAAA,gBAAA,CACT,QAAS,4DACX,CACF,EAEaC,CAAkB,CAAA,CAC7BC,CACAC,CAAAA,CAAAA,GAEID,CAAcF,IAAAA,CAAAA,CACT,IAAIF,CAAc,CAAA,CACvB,GAAGE,CACDE,CAAAA,CACF,EACA,OACEF,CAAAA,CAAAA,CAAsBE,CAAgD,CACnE,CAAA,OAAA,CACL,QACEC,CACAH,EAAAA,CAAAA,CAAsBE,CAAgD,CACnE,CAAA,OACP,CAAC,CAII,CAAA,IAAIJ,CAAc,CAAA,CACvB,KAAO,CAAA,gBAAA,CACP,QAAS,8BACT,CAAA,OAAA,CAASK,CACX,CAAC,CAAA,KC9CUC,CAAN,CAAA,KAAiB,CACZ,MAEV,CAAA,WAAA,CAAYC,EAAwB,CAClC,IAAA,CAAK,OAASA,EAChB,CAEA,MAAM,KACJC,CAAAA,CAAAA,CACAC,EACY,CAEZ,IAAIC,EAAM,CADM,EAAA,IAAA,CAAK,OAAO,OAAW,EAAA,4BACnB,GAClBF,CAAS,CAAA,UAAA,CAAW,GAAG,CAAIA,CAAAA,CAAAA,CAAW,IAAIA,CAAQ,CAAA,CACpD,GAEA,GAAIC,CAAAA,CAAQ,CACV,IAAME,CAAAA,CAAc,IAAI,eAAA,CACxB,MAAO,CAAA,OAAA,CAAQF,CAAM,CAAE,CAAA,OAAA,CAAQ,CAAC,CAACG,CAAAA,CAAKC,CAAK,CAAM,GAAA,CACpBA,GAAU,IACnCF,EAAAA,CAAAA,CAAY,OAAOC,CAAK,CAAA,MAAA,CAAOC,CAAK,CAAC,EAEzC,CAAC,CACDH,CAAAA,CAAAA,EAAO,CAAIC,CAAAA,EAAAA,CAAAA,CAAY,QAAS,EAAC,GACnC,CAEA,IAAMG,EAAW,MAAM,KAAA,CAAMJ,EAAK,CAChC,MAAA,CAAQ,MACR,OAAS,CAAA,CACP,YAAa,IAAK,CAAA,MAAA,CAAO,OACzB,cAAgB,CAAA,kBAClB,CACF,CAAC,CAAA,CAEKK,CAAcD,CAAAA,CAAAA,CAAS,OAAQ,CAAA,GAAA,CAAI,cAAc,CAEvD,CAAA,GAAI,CAACA,CAAS,CAAA,EAAA,CAAI,CAEhB,GAAI,CAAC,IAAK,GAAK,CAAA,GAAG,EAAE,QAASA,CAAAA,CAAAA,CAAS,MAAM,CAAG,CAAA,CAC7C,IAAME,CAAY,CAAA,MAAMF,EAAS,IAAK,EAAA,CACtC,MAAMX,CAAgBW,CAAAA,CAAAA,CAAS,OAAQE,CAAU,CAAA,OAAO,CAC1D,CAGA,GAAID,GAAa,QAAS,CAAA,kBAAkB,EAAG,CAC7C,IAAMC,EAAY,MAAMF,CAAAA,CAAS,MACjC,CAAA,MAAM,IAAId,CAAAA,CAAcgB,CAAS,CACnC,MAEQ,MAAA,IAAIhB,EAAc,CACtB,KAAA,CAAO,iBACP,OAAS,CAAA,kCAAA,CACT,QAAS,MAAMc,CAAAA,CAAS,MAC1B,CAAC,CAEL,CAEA,GAAI,CACF,GAAI,CAACC,CAAa,EAAA,QAAA,CAAS,kBAAkB,CAAA,CAC3C,MAAM,IAAIf,CAAAA,CAAc,CACtB,KAAO,CAAA,gBAAA,CACP,QAAS,yBACT,CAAA,OAAA,CAAS,4DACX,CAAC,CAAA,CAGH,OAAQ,MAAMc,CAAAA,CAAS,MACzB,CAAA,MAASb,EAAO,CACd,MAAM,IAAID,CAAAA,CAAc,CACtB,KAAA,CAAO,iBACP,OAAS,CAAA,0BAAA,CACT,QAASC,CAAiB,YAAA,KAAA,CAAQA,EAAM,OAAU,CAAA,eACpD,CAAC,CACH,CACF,CACF,ECjEO,IAAMgB,EAAN,cAA6BX,CAAW,CAC7C,MAAM,UAAA,CAAWG,EAA+C,CAC9D,OAAO,KAAK,KAAkB,CAAA,qBAAA,CAAuBA,CAAM,CAC7D,CAEA,MAAM,SAAUA,CAAAA,CAAAA,CAAwD,CACtE,OAAO,IAAA,CAAK,MACV,+BACAA,CAAAA,CACF,CACF,CACF,MCrBaS,CAAN,CAAA,cAAyBZ,CAAW,CACzC,MAAM,MAAA,CAAOI,EAA8B,CACzC,OAAO,KAAK,KAAc,CAAA,aAAA,CAAe,CAAE,GAAAA,CAAAA,CAAI,CAAC,CAClD,CAEA,MAAM,GAAIA,CAAAA,CAAAA,CAA2B,CACnC,OAAO,IAAA,CAAK,MAAW,UAAY,CAAA,CAAE,GAAAA,CAAAA,CAAI,CAAC,CAC5C,CACF,ECFO,IAAMS,EAAN,KAAe,CACX,QACA,GAET,CAAA,WAAA,CAAYZ,EAAwB,CAClC,IAAA,CAAK,QAAU,IAAIU,CAAAA,CAAeV,CAAM,CACxC,CAAA,IAAA,CAAK,IAAM,IAAIW,CAAAA,CAAWX,CAAM,EAClC,CACF","file":"index.cjs","sourcesContent":["export interface TranscriptChunk {\n  text: string;\n  offset: number;\n  duration: number;\n  lang: string;\n}\n\nexport interface Transcript {\n  content: TranscriptChunk[] | string;\n  lang: string;\n  availableLangs: string[];\n}\n\nexport interface TranslatedTranscript {\n  content: TranscriptChunk[] | string;\n  lang: string;\n}\n\nexport interface Scrape {\n  url: string;\n  content: string;\n  name: string;\n  description: string;\n  ogUrl: string;\n  countCharacters: number;\n  urls: string[];\n}\n\nexport interface Map {\n  urls: string[];\n}\n\nexport interface SupadataConfig {\n  apiKey: string;\n  baseUrl?: string;\n}\n\nexport class SupadataError extends Error {\n  error:\n    | 'invalid-request'\n    | 'missing-parameters'\n    | 'internal-error'\n    | 'transcript-unavailable'\n    | 'video-not-found'\n    | 'video-id-invalid'\n    | 'youtube-api-error'\n    | 'limit-exceeded';\n  details: string;\n  documentationUrl: string;\n\n  constructor(error: {\n    error: SupadataError['error'];\n    message?: string;\n    details?: string;\n    documentationUrl?: string;\n  }) {\n    super(error.message || 'An unexpected error occurred');\n    this.error = error.error || 'internal-error';\n    this.details = error.details || 'An unexpected error occurred';\n    this.documentationUrl = error.documentationUrl || '';\n    this.name = 'SupadataError';\n  }\n}\n","import { SupadataError } from './types';\n\n/*\nThe API gateway returns errors in text/plain content type. \nAs a temporary workaround we're mapping them to SupadataError.\n*/\n\nconst GATEWAY_STATUS_ERRORS = {\n  403: {\n    error: 'invalid-request' as const,\n    message: 'Invalid or missing API key',\n    details: 'Please ensure you have provided a valid API key',\n  },\n  404: {\n    error: 'invalid-request' as const,\n    message: 'Endpoint does not exist',\n    details: 'The API endpoint you are trying to access does not exist',\n  },\n  429: {\n    error: 'limit-exceeded' as const,\n    message: 'Limit exceeded',\n    details: 'You have exceeded the allowed request rate or quota limits',\n  },\n};\n\nexport const mapGatewayError = (\n  statusCode: number,\n  errorText: string\n): SupadataError => {\n  if (statusCode in GATEWAY_STATUS_ERRORS) {\n    return new SupadataError({\n      ...GATEWAY_STATUS_ERRORS[\n        statusCode as keyof typeof GATEWAY_STATUS_ERRORS\n      ],\n      message:\n        GATEWAY_STATUS_ERRORS[statusCode as keyof typeof GATEWAY_STATUS_ERRORS]\n          .message,\n      details:\n        errorText ||\n        GATEWAY_STATUS_ERRORS[statusCode as keyof typeof GATEWAY_STATUS_ERRORS]\n          .details,\n    });\n  }\n\n  // Default error if status code is not recognized\n  return new SupadataError({\n    error: 'internal-error',\n    message: 'An unexpected error occurred',\n    details: errorText,\n  });\n};\n","import { SupadataConfig, SupadataError } from './types.js';\nimport { mapGatewayError } from './gateway-error-mapper.js';\n\nexport class BaseClient {\n  protected config: SupadataConfig;\n\n  constructor(config: SupadataConfig) {\n    this.config = config;\n  }\n\n  async fetch<T>(\n    endpoint: string,\n    params: Record<string, unknown> | object\n  ): Promise<T> {\n    const baseUrl = this.config.baseUrl || 'https://api.supadata.ai/v1';\n    let url = `${baseUrl}${\n      endpoint.startsWith('/') ? endpoint : `/${endpoint}`\n    }`;\n\n    if (params) {\n      const queryParams = new URLSearchParams();\n      Object.entries(params).forEach(([key, value]) => {\n        if (value !== undefined && value !== null) {\n          queryParams.append(key, String(value));\n        }\n      });\n      url += `?${queryParams.toString()}`;\n    }\n\n    const response = await fetch(url, {\n      method: 'GET',\n      headers: {\n        'x-api-key': this.config.apiKey,\n        'Content-Type': 'application/json',\n      },\n    });\n\n    const contentType = response.headers.get('content-type');\n\n    if (!response.ok) {\n      // First check for gateway-specific status codes\n      if ([403, 404, 429].includes(response.status)) {\n        const errorData = await response.json();\n        throw mapGatewayError(response.status, errorData.message);\n      }\n\n      // Handle standard API errors\n      if (contentType?.includes('application/json')) {\n        const errorData = await response.json();\n        throw new SupadataError(errorData);\n      } else {\n        // Fallback for unexpected non-JSON errors\n        throw new SupadataError({\n          error: 'internal-error',\n          message: 'Unexpected error response format',\n          details: await response.text(),\n        });\n      }\n    }\n\n    try {\n      if (!contentType?.includes('application/json')) {\n        throw new SupadataError({\n          error: 'internal-error',\n          message: 'Invalid response format',\n          details: 'Expected JSON response but received different content type',\n        });\n      }\n\n      return (await response.json()) as T;\n    } catch (error) {\n      throw new SupadataError({\n        error: 'internal-error',\n        message: 'Failed to parse response',\n        details: error instanceof Error ? error.message : 'Unknown error',\n      });\n    }\n  }\n}\n","import { BaseClient } from '../client.js';\nimport { Transcript, TranslatedTranscript } from '../types.js';\n\nexport interface TranscriptParams {\n  videoId: string;\n  lang?: string;\n  text?: boolean;\n}\n\nexport interface TranslateParams extends TranscriptParams {\n  lang: string;\n}\n\nexport class YouTubeService extends BaseClient {\n  async transcript(params: TranscriptParams): Promise<Transcript> {\n    return this.fetch<Transcript>('/youtube/transcript', params);\n  }\n\n  async translate(params: TranslateParams): Promise<TranslatedTranscript> {\n    return this.fetch<TranslatedTranscript>(\n      '/youtube/transcript/translate',\n      params\n    );\n  }\n}\n","import { BaseClient } from '../client.js';\nimport { Scrape, Map } from '../types.js';\n\nexport class WebService extends BaseClient {\n  async scrape(url: string): Promise<Scrape> {\n    return this.fetch<Scrape>('/web/scrape', { url });\n  }\n\n  async map(url: string): Promise<Map> {\n    return this.fetch<Map>('/web/map', { url });\n  }\n}\n","import { SupadataConfig } from './types.js';\nimport { YouTubeService } from './services/youtube.js';\nimport { WebService } from './services/web.js';\n\nexport * from './types.js';\nexport * from './client.js';\nexport * from './services/youtube.js';\nexport * from './services/web.js';\n\nexport class Supadata {\n  readonly youtube: YouTubeService;\n  readonly web: WebService;\n\n  constructor(config: SupadataConfig) {\n    this.youtube = new YouTubeService(config);\n    this.web = new WebService(config);\n  }\n}\n"]}
+{"version":3,"sources":["../src/types.ts","../src/gateway-error-mapper.ts","../src/client.ts","../src/services/youtube.ts","../src/services/web.ts","../src/index.ts"],"names":["SupadataError","error","GATEWAY_STATUS_ERRORS","mapGatewayError","statusCode","errorText","BaseClient","config","endpoint","params","method","url","queryParams","key","value","body","options","response","contentType","errorData","YouTubeService","WebService","request","jobId","pages","nextUrl","Supadata"],"mappings":"aAoDaA,IAAAA,CAAAA,CAAN,cAA4B,KAAM,CACvC,KAAA,CASA,OACA,CAAA,gBAAA,CAEA,WAAYC,CAAAA,CAAAA,CAKT,CACD,KAAA,CAAMA,CAAM,CAAA,OAAA,EAAW,8BAA8B,CACrD,CAAA,IAAA,CAAK,KAAQA,CAAAA,CAAAA,CAAM,KAAS,EAAA,gBAAA,CAC5B,IAAK,CAAA,OAAA,CAAUA,CAAM,CAAA,OAAA,EAAW,8BAChC,CAAA,IAAA,CAAK,gBAAmBA,CAAAA,CAAAA,CAAM,kBAAoB,EAClD,CAAA,IAAA,CAAK,IAAO,CAAA,gBACd,CACF,ECtEA,IAAMC,CAAAA,CAAwB,CAC5B,GAAA,CAAK,CACH,KAAA,CAAO,iBACP,CAAA,OAAA,CAAS,6BACT,OAAS,CAAA,iDACX,CACA,CAAA,GAAA,CAAK,CACH,KAAA,CAAO,iBACP,CAAA,OAAA,CAAS,yBACT,CAAA,OAAA,CAAS,0DACX,CAAA,CACA,GAAK,CAAA,CACH,MAAO,gBACP,CAAA,OAAA,CAAS,gBACT,CAAA,OAAA,CAAS,4DACX,CACF,CAEaC,CAAAA,CAAAA,CAAkB,CAC7BC,CAAAA,CACAC,CAEID,GAAAA,CAAAA,IAAcF,CACT,CAAA,IAAIF,EAAc,CACvB,GAAGE,CACDE,CAAAA,CACF,CACA,CAAA,OAAA,CACEF,CAAsBE,CAAAA,CAAgD,CACnE,CAAA,OAAA,CACL,OACEC,CAAAA,CAAAA,EACAH,CAAsBE,CAAAA,CAAgD,EACnE,OACP,CAAC,CAII,CAAA,IAAIJ,CAAc,CAAA,CACvB,KAAO,CAAA,gBAAA,CACP,OAAS,CAAA,8BAAA,CACT,OAASK,CAAAA,CACX,CAAC,CAAA,CC9CUC,IAAAA,CAAAA,CAAN,KAAiB,CACZ,MAEV,CAAA,WAAA,CAAYC,CAAwB,CAAA,CAClC,IAAK,CAAA,MAAA,CAASA,EAChB,CAEA,MAAgB,KAAA,CACdC,CACAC,CAAAA,CAAAA,CAA8B,EAC9BC,CAAAA,CAAAA,CAAyB,KACb,CAAA,CAEZ,IAAIC,CAAAA,CAAM,CADM,EAAA,IAAA,CAAK,MAAO,CAAA,OAAA,EAAW,4BACnB,CAAA,EAClBH,CAAS,CAAA,UAAA,CAAW,GAAG,CAAIA,CAAAA,CAAAA,CAAW,CAAIA,CAAAA,EAAAA,CAAQ,CACpD,CAAA,CAAA,CAAA,CAEA,GAAIE,CAAAA,GAAW,KAAS,EAAA,MAAA,CAAO,IAAKD,CAAAA,CAAM,CAAE,CAAA,MAAA,CAAS,EAAG,CACtD,IAAMG,CAAc,CAAA,IAAI,eACxB,CAAA,MAAA,CAAO,OAAQH,CAAAA,CAAM,CAAE,CAAA,OAAA,CAAQ,CAAC,CAACI,CAAKC,CAAAA,CAAK,CAAM,GAAA,CACpBA,CAAU,EAAA,IAAA,EACnCF,CAAY,CAAA,MAAA,CAAOC,CAAK,CAAA,MAAA,CAAOC,CAAK,CAAC,EAEzC,CAAC,CACDH,CAAAA,CAAAA,EAAO,CAAIC,CAAAA,EAAAA,CAAAA,CAAY,UAAU,CAAA,EACnC,CAEA,OAAO,IAAK,CAAA,QAAA,CAAYD,CAAKD,CAAAA,CAAAA,CAAQD,CAAM,CAC7C,CAEA,MAAgB,QACdE,CAAAA,CAAAA,CACAD,EAAyB,KACzBK,CAAAA,CAAAA,CACY,CACZ,IAAMC,CAAuB,CAAA,CAC3B,MAAAN,CAAAA,CAAAA,CACA,OAAS,CAAA,CACP,WAAa,CAAA,IAAA,CAAK,MAAO,CAAA,MAAA,CACzB,eAAgB,kBAClB,CACF,CAEIA,CAAAA,CAAAA,GAAW,MAAUK,EAAAA,CAAAA,GACvBC,CAAQ,CAAA,IAAA,CAAO,IAAK,CAAA,SAAA,CAAUD,CAAI,CAAA,CAAA,CAGpC,IAAME,CAAAA,CAAW,MAAM,KAAA,CAAMN,CAAKK,CAAAA,CAAO,CAEnCE,CAAAA,CAAAA,CAAcD,CAAS,CAAA,OAAA,CAAQ,GAAI,CAAA,cAAc,CAEvD,CAAA,GAAI,CAACA,CAAAA,CAAS,EAAI,CAAA,CAEhB,GAAI,CAAC,GAAA,CAAK,GAAK,CAAA,GAAG,CAAE,CAAA,QAAA,CAASA,CAAS,CAAA,MAAM,CAAG,CAAA,CAC7C,IAAME,CAAAA,CAAY,MAAMF,CAAAA,CAAS,MACjC,CAAA,MAAMd,CAAgBc,CAAAA,CAAAA,CAAS,MAAQE,CAAAA,CAAAA,CAAU,OAAO,CAC1D,CAGA,GAAID,CAAa,EAAA,QAAA,CAAS,kBAAkB,CAAA,CAAG,CAC7C,IAAMC,CAAAA,CAAY,MAAMF,CAAAA,CAAS,IAAK,EAAA,CACtC,MAAM,IAAIjB,CAAcmB,CAAAA,CAAS,CACnC,CAAA,KAEQ,MAAA,IAAInB,CAAc,CAAA,CACtB,KAAO,CAAA,gBAAA,CACP,OAAS,CAAA,kCAAA,CACT,OAAS,CAAA,MAAMiB,CAAS,CAAA,IAAA,EAC1B,CAAC,CAEL,CAEA,GAAI,CACF,GAAI,CAACC,CAAAA,EAAa,QAAS,CAAA,kBAAkB,CAC3C,CAAA,MAAM,IAAIlB,CAAAA,CAAc,CACtB,KAAA,CAAO,gBACP,CAAA,OAAA,CAAS,yBACT,CAAA,OAAA,CAAS,4DACX,CAAC,CAAA,CAGH,OAAQ,MAAMiB,CAAS,CAAA,IAAA,EACzB,CAAA,MAAShB,CAAO,CAAA,CACd,MAAM,IAAID,CAAc,CAAA,CACtB,MAAO,gBACP,CAAA,OAAA,CAAS,0BACT,CAAA,OAAA,CAASC,CAAiB,YAAA,KAAA,CAAQA,CAAM,CAAA,OAAA,CAAU,eACpD,CAAC,CACH,CACF,CACF,ECtEamB,IAAAA,CAAAA,CAAN,cAA6Bd,CAAW,CAW7C,MAAM,UAAWG,CAAAA,CAAAA,CAA+C,CAC9D,OAAO,IAAK,CAAA,KAAA,CAAkB,qBAAuBA,CAAAA,CAAM,CAC7D,CAYA,MAAM,SAAUA,CAAAA,CAAAA,CAAwD,CACtE,OAAO,IAAK,CAAA,KAAA,CACV,+BACAA,CAAAA,CACF,CACF,CACF,ECnDO,IAAMY,CAAN,CAAA,cAAyBf,CAAW,CAOzC,MAAM,MAAOK,CAAAA,CAAAA,CAA8B,CACzC,OAAO,IAAK,CAAA,KAAA,CAAc,aAAe,CAAA,CAAE,GAAAA,CAAAA,CAAI,CAAC,CAClD,CAQA,MAAM,GAAA,CAAIA,CAA2B,CAAA,CACnC,OAAO,IAAA,CAAK,KAAW,CAAA,UAAA,CAAY,CAAE,GAAA,CAAAA,CAAI,CAAC,CAC5C,CAUA,MAAM,KAAA,CAAMW,CAAuC,CAAA,CACjD,OAAO,IAAA,CAAK,KAAa,CAAA,YAAA,CAAcA,CAAS,CAAA,MAAM,CACxD,CASA,MAAM,eAAA,CAAgBC,CAAkC,CAAA,CACtD,IAAIN,CACAO,CAAAA,CAAAA,CAAkB,EAAC,CACnBC,CAEJ,CAAA,GACER,CAAW,CAAA,MAAOQ,CACd,CAAA,IAAA,CAAK,QAAmBA,CAAAA,CAAO,CAC/B,CAAA,IAAA,CAAK,MAAgB,CAAcF,WAAAA,EAAAA,CAAK,CAAE,CAAA,CAAA,CAAA,CAE1CN,CAAS,CAAA,KAAA,GACXO,CAAQ,CAAA,CAAC,GAAGA,CAAAA,CAAO,GAAGP,CAAAA,CAAS,KAAK,CAAA,CAAA,CAEtCQ,EAAUR,CAAS,CAAA,IAAA,CAAA,MACZQ,CAET,EAAA,OAAOR,CACT,CACF,ECpDO,IAAMS,CAAN,CAAA,KAAe,CACX,OAAA,CACA,GAET,CAAA,WAAA,CAAYnB,CAAwB,CAAA,CAClC,IAAK,CAAA,OAAA,CAAU,IAAIa,CAAAA,CAAeb,CAAM,CAAA,CACxC,IAAK,CAAA,GAAA,CAAM,IAAIc,CAAAA,CAAWd,CAAM,EAClC,CACF","file":"index.cjs","sourcesContent":["export interface TranscriptChunk {\n  text: string;\n  offset: number;\n  duration: number;\n  lang: string;\n}\n\nexport interface Transcript {\n  content: TranscriptChunk[] | string;\n  lang: string;\n  availableLangs: string[];\n}\n\nexport interface TranslatedTranscript {\n  content: TranscriptChunk[] | string;\n  lang: string;\n}\n\nexport interface Scrape {\n  url: string;\n  content: string;\n  name: string;\n  description: string;\n  ogUrl: string;\n  countCharacters: number;\n  urls: string[];\n}\n\nexport interface Map {\n  urls: string[];\n}\n\nexport interface CrawlRequest {\n  url: string;\n  limit?: number;\n}\n\nexport interface Crawl {\n  jobId: string;\n}\n\nexport interface CrawlJob {\n  status: 'scraping' | 'completed' | 'failed' | 'cancelled';\n  pages?: Scrape[];\n  next?: string;\n}\n\nexport interface SupadataConfig {\n  apiKey: string;\n  baseUrl?: string;\n}\n\nexport class SupadataError extends Error {\n  error:\n    | 'invalid-request'\n    | 'missing-parameters'\n    | 'internal-error'\n    | 'transcript-unavailable'\n    | 'video-not-found'\n    | 'video-id-invalid'\n    | 'youtube-api-error'\n    | 'limit-exceeded';\n  details: string;\n  documentationUrl: string;\n\n  constructor(error: {\n    error: SupadataError['error'];\n    message?: string;\n    details?: string;\n    documentationUrl?: string;\n  }) {\n    super(error.message || 'An unexpected error occurred');\n    this.error = error.error || 'internal-error';\n    this.details = error.details || 'An unexpected error occurred';\n    this.documentationUrl = error.documentationUrl || '';\n    this.name = 'SupadataError';\n  }\n}\n","import { SupadataError } from './types';\n\n/*\nThe API gateway returns errors in text/plain content type. \nAs a temporary workaround we're mapping them to SupadataError.\n*/\n\nconst GATEWAY_STATUS_ERRORS = {\n  403: {\n    error: 'invalid-request' as const,\n    message: 'Invalid or missing API key',\n    details: 'Please ensure you have provided a valid API key',\n  },\n  404: {\n    error: 'invalid-request' as const,\n    message: 'Endpoint does not exist',\n    details: 'The API endpoint you are trying to access does not exist',\n  },\n  429: {\n    error: 'limit-exceeded' as const,\n    message: 'Limit exceeded',\n    details: 'You have exceeded the allowed request rate or quota limits',\n  },\n};\n\nexport const mapGatewayError = (\n  statusCode: number,\n  errorText: string\n): SupadataError => {\n  if (statusCode in GATEWAY_STATUS_ERRORS) {\n    return new SupadataError({\n      ...GATEWAY_STATUS_ERRORS[\n        statusCode as keyof typeof GATEWAY_STATUS_ERRORS\n      ],\n      message:\n        GATEWAY_STATUS_ERRORS[statusCode as keyof typeof GATEWAY_STATUS_ERRORS]\n          .message,\n      details:\n        errorText ||\n        GATEWAY_STATUS_ERRORS[statusCode as keyof typeof GATEWAY_STATUS_ERRORS]\n          .details,\n    });\n  }\n\n  // Default error if status code is not recognized\n  return new SupadataError({\n    error: 'internal-error',\n    message: 'An unexpected error occurred',\n    details: errorText,\n  });\n};\n","import { SupadataConfig, SupadataError } from './types.js';\nimport { mapGatewayError } from './gateway-error-mapper.js';\n\nexport class BaseClient {\n  protected config: SupadataConfig;\n\n  constructor(config: SupadataConfig) {\n    this.config = config;\n  }\n\n  protected async fetch<T>(\n    endpoint: string,\n    params: Record<string, any> = {},\n    method: 'GET' | 'POST' = 'GET'\n  ): Promise<T> {\n    const baseUrl = this.config.baseUrl || 'https://api.supadata.ai/v1';\n    let url = `${baseUrl}${\n      endpoint.startsWith('/') ? endpoint : `/${endpoint}`\n    }`;\n\n    if (method === 'GET' && Object.keys(params).length > 0) {\n      const queryParams = new URLSearchParams();\n      Object.entries(params).forEach(([key, value]) => {\n        if (value !== undefined && value !== null) {\n          queryParams.append(key, String(value));\n        }\n      });\n      url += `?${queryParams.toString()}`;\n    }\n\n    return this.fetchUrl<T>(url, method, params);\n  }\n\n  protected async fetchUrl<T>(\n    url: string,\n    method: 'GET' | 'POST' = 'GET',\n    body?: Record<string, any>\n  ): Promise<T> {\n    const options: RequestInit = {\n      method,\n      headers: {\n        'x-api-key': this.config.apiKey,\n        'Content-Type': 'application/json',\n      },\n    };\n\n    if (method === 'POST' && body) {\n      options.body = JSON.stringify(body);\n    }\n\n    const response = await fetch(url, options);\n\n    const contentType = response.headers.get('content-type');\n\n    if (!response.ok) {\n      // First check for gateway-specific status codes\n      if ([403, 404, 429].includes(response.status)) {\n        const errorData = await response.json();\n        throw mapGatewayError(response.status, errorData.message);\n      }\n\n      // Handle standard API errors\n      if (contentType?.includes('application/json')) {\n        const errorData = await response.json();\n        throw new SupadataError(errorData);\n      } else {\n        // Fallback for unexpected non-JSON errors\n        throw new SupadataError({\n          error: 'internal-error',\n          message: 'Unexpected error response format',\n          details: await response.text(),\n        });\n      }\n    }\n\n    try {\n      if (!contentType?.includes('application/json')) {\n        throw new SupadataError({\n          error: 'internal-error',\n          message: 'Invalid response format',\n          details: 'Expected JSON response but received different content type',\n        });\n      }\n\n      return (await response.json()) as T;\n    } catch (error) {\n      throw new SupadataError({\n        error: 'internal-error',\n        message: 'Failed to parse response',\n        details: error instanceof Error ? error.message : 'Unknown error',\n      });\n    }\n  }\n}\n","import { BaseClient } from '../client.js';\nimport { Transcript, TranslatedTranscript } from '../types.js';\n\n/**\n * Ensures exactly one property from the specified keys is provided.\n * @example\n * // Valid: { url: \"...\" } or { videoId: \"...\" }\n * // Invalid: {} or { url: \"...\", videoId: \"...\" }\n */\ntype ExactlyOne<T, Keys extends keyof T> = {\n  [K in Keys]: { [P in K]-?: T[P] } & { [P in Exclude<Keys, K>]?: never };\n}[Keys] &\n  Omit<T, Keys>;\n\nexport type TranscriptParams = {\n  lang?: string;\n  text?: boolean;\n} & ExactlyOne<{ videoId: string; url: string }, 'videoId' | 'url'>;\n\nexport interface TranslateParams extends Omit<TranscriptParams, 'lang'> {\n  lang: string;\n}\n\nexport class YouTubeService extends BaseClient {\n  /**\n   * Fetches a transcript for a YouTube video.\n   *\n   * @param params - Parameters for fetching the transcript\n   * @param params.videoId - The YouTube video ID (provide either this OR url)\n   * @param params.url - The YouTube video URL (provide either this OR videoId)\n   * @param params.lang - Optional language code for the transcript\n   * @param params.text - Optional flag to return plain text instead of timestamped list\n   * @returns A promise that resolves to the video transcript\n   */\n  async transcript(params: TranscriptParams): Promise<Transcript> {\n    return this.fetch<Transcript>('/youtube/transcript', params);\n  }\n\n  /**\n   * Translates a YouTube video transcript to a specified language.\n   *\n   * @param params - Parameters for translating the transcript\n   * @param params.videoId - The YouTube video ID (provide either this OR url)\n   * @param params.url - The YouTube video URL (provide either this OR videoId)\n   * @param params.lang - The target language code for translation\n   * @param params.text - Optional flag to return plain text instead of timestamped list\n   * @returns A promise that resolves to the translated transcript\n   */\n  async translate(params: TranslateParams): Promise<TranslatedTranscript> {\n    return this.fetch<TranslatedTranscript>(\n      '/youtube/transcript/translate',\n      params\n    );\n  }\n}\n","import { BaseClient } from '../client.js';\nimport { Scrape, Map, CrawlRequest, Crawl, CrawlJob } from '../types.js';\n\nexport class WebService extends BaseClient {\n  /**\n   * Extract content from any web page to Markdown format.\n   *\n   * @param url - URL of the webpage to scrape\n   * @returns A promise that resolves to the scraped content\n   */\n  async scrape(url: string): Promise<Scrape> {\n    return this.fetch<Scrape>('/web/scrape', { url });\n  }\n\n  /**\n   * Extract all links found on a webpage.\n   *\n   * @param url - URL of the webpage to map\n   * @returns A promise that resolves to a map of URLs found on the page\n   */\n  async map(url: string): Promise<Map> {\n    return this.fetch<Map>('/web/map', { url });\n  }\n\n  /**\n   * Create a crawl job to extract content from all pages on a website.\n   *\n   * @param request - Crawl request parameters\n   * @param request.url - URL of the website to crawl\n   * @param request.limit - Maximum number of pages to crawl (default: 100, max: 5000)\n   * @returns A promise that resolves to the crawl job id\n   */\n  async crawl(request: CrawlRequest): Promise<Crawl> {\n    return this.fetch<Crawl>('/web/crawl', request, 'POST');\n  }\n\n  /**\n   * Get the status and results of a crawl job.\n   * Automatically handles pagination to retrieve all pages from the crawl.\n   *\n   * @param jobId - The ID of the crawl job to retrieve\n   * @returns A promise that resolves to the complete crawl job results\n   */\n  async getCrawlResults(jobId: string): Promise<CrawlJob> {\n    let response: CrawlJob;\n    let pages: Scrape[] = [];\n    let nextUrl: string | undefined;\n\n    do {\n      response = await (nextUrl\n        ? this.fetchUrl<CrawlJob>(nextUrl)\n        : this.fetch<CrawlJob>(`/web/crawl/${jobId}`));\n\n      if (response.pages) {\n        pages = [...pages, ...response.pages];\n      }\n      nextUrl = response.next;\n    } while (nextUrl);\n\n    return response;\n  }\n}\n","import { SupadataConfig } from './types.js';\nimport { YouTubeService } from './services/youtube.js';\nimport { WebService } from './services/web.js';\n\nexport * from './types.js';\nexport * from './client.js';\nexport * from './services/youtube.js';\nexport * from './services/web.js';\n\nexport class Supadata {\n  readonly youtube: YouTubeService;\n  readonly web: WebService;\n\n  constructor(config: SupadataConfig) {\n    this.youtube = new YouTubeService(config);\n    this.web = new WebService(config);\n  }\n}\n"]}

package/dist/index.d.mts

@@ -25,6 +25,18 @@ interface Scrape {
 interface Map {
     urls: string[];
 }
+interface CrawlRequest {
+    url: string;
+    limit?: number;
+}
+interface Crawl {
+    jobId: string;
+}
+interface CrawlJob {
+    status: 'scraping' | 'completed' | 'failed' | 'cancelled';
+    pages?: Scrape[];
+    next?: string;
+}
 interface SupadataConfig {
     apiKey: string;
     baseUrl?: string;
@@ -44,25 +56,90 @@ declare class SupadataError extends Error {
 declare class BaseClient {
     protected config: SupadataConfig;
     constructor(config: SupadataConfig);
-    fetch<T>(endpoint: string, params: Record<string, unknown> | object): Promise<T>;
+    protected fetch<T>(endpoint: string, params?: Record<string, any>, method?: 'GET' | 'POST'): Promise<T>;
+    protected fetchUrl<T>(url: string, method?: 'GET' | 'POST', body?: Record<string, any>): Promise<T>;
 }
 
-interface TranscriptParams {
-    videoId: string;
+/**
+ * Ensures exactly one property from the specified keys is provided.
+ * @example
+ * // Valid: { url: "..." } or { videoId: "..." }
+ * // Invalid: {} or { url: "...", videoId: "..." }
+ */
+type ExactlyOne<T, Keys extends keyof T> = {
+    [K in Keys]: {
+        [P in K]-?: T[P];
+    } & {
+        [P in Exclude<Keys, K>]?: never;
+    };
+}[Keys] & Omit<T, Keys>;
+type TranscriptParams = {
     lang?: string;
     text?: boolean;
-}
-interface TranslateParams extends TranscriptParams {
+} & ExactlyOne<{
+    videoId: string;
+    url: string;
+}, 'videoId' | 'url'>;
+interface TranslateParams extends Omit<TranscriptParams, 'lang'> {
     lang: string;
 }
 declare class YouTubeService extends BaseClient {
+    /**
+     * Fetches a transcript for a YouTube video.
+     *
+     * @param params - Parameters for fetching the transcript
+     * @param params.videoId - The YouTube video ID (provide either this OR url)
+     * @param params.url - The YouTube video URL (provide either this OR videoId)
+     * @param params.lang - Optional language code for the transcript
+     * @param params.text - Optional flag to return plain text instead of timestamped list
+     * @returns A promise that resolves to the video transcript
+     */
     transcript(params: TranscriptParams): Promise<Transcript>;
+    /**
+     * Translates a YouTube video transcript to a specified language.
+     *
+     * @param params - Parameters for translating the transcript
+     * @param params.videoId - The YouTube video ID (provide either this OR url)
+     * @param params.url - The YouTube video URL (provide either this OR videoId)
+     * @param params.lang - The target language code for translation
+     * @param params.text - Optional flag to return plain text instead of timestamped list
+     * @returns A promise that resolves to the translated transcript
+     */
     translate(params: TranslateParams): Promise<TranslatedTranscript>;
 }
 
 declare class WebService extends BaseClient {
+    /**
+     * Extract content from any web page to Markdown format.
+     *
+     * @param url - URL of the webpage to scrape
+     * @returns A promise that resolves to the scraped content
+     */
     scrape(url: string): Promise<Scrape>;
+    /**
+     * Extract all links found on a webpage.
+     *
+     * @param url - URL of the webpage to map
+     * @returns A promise that resolves to a map of URLs found on the page
+     */
     map(url: string): Promise<Map>;
+    /**
+     * Create a crawl job to extract content from all pages on a website.
+     *
+     * @param request - Crawl request parameters
+     * @param request.url - URL of the website to crawl
+     * @param request.limit - Maximum number of pages to crawl (default: 100, max: 5000)
+     * @returns A promise that resolves to the crawl job id
+     */
+    crawl(request: CrawlRequest): Promise<Crawl>;
+    /**
+     * Get the status and results of a crawl job.
+     * Automatically handles pagination to retrieve all pages from the crawl.
+     *
+     * @param jobId - The ID of the crawl job to retrieve
+     * @returns A promise that resolves to the complete crawl job results
+     */
+    getCrawlResults(jobId: string): Promise<CrawlJob>;
 }
 
 declare class Supadata {
@@ -71,4 +148,4 @@ declare class Supadata {
     constructor(config: SupadataConfig);
 }
 
-export { BaseClient, type Map, type Scrape, Supadata, type SupadataConfig, SupadataError, type Transcript, type TranscriptChunk, type TranscriptParams, type TranslateParams, type TranslatedTranscript, WebService, YouTubeService };
+export { BaseClient, type Crawl, type CrawlJob, type CrawlRequest, type Map, type Scrape, Supadata, type SupadataConfig, SupadataError, type Transcript, type TranscriptChunk, type TranscriptParams, type TranslateParams, type TranslatedTranscript, WebService, YouTubeService };

package/dist/index.d.ts

@@ -25,6 +25,18 @@ interface Scrape {
 interface Map {
     urls: string[];
 }
+interface CrawlRequest {
+    url: string;
+    limit?: number;
+}
+interface Crawl {
+    jobId: string;
+}
+interface CrawlJob {
+    status: 'scraping' | 'completed' | 'failed' | 'cancelled';
+    pages?: Scrape[];
+    next?: string;
+}
 interface SupadataConfig {
     apiKey: string;
     baseUrl?: string;
@@ -44,25 +56,90 @@ declare class SupadataError extends Error {
 declare class BaseClient {
     protected config: SupadataConfig;
     constructor(config: SupadataConfig);
-    fetch<T>(endpoint: string, params: Record<string, unknown> | object): Promise<T>;
+    protected fetch<T>(endpoint: string, params?: Record<string, any>, method?: 'GET' | 'POST'): Promise<T>;
+    protected fetchUrl<T>(url: string, method?: 'GET' | 'POST', body?: Record<string, any>): Promise<T>;
 }
 
-interface TranscriptParams {
-    videoId: string;
+/**
+ * Ensures exactly one property from the specified keys is provided.
+ * @example
+ * // Valid: { url: "..." } or { videoId: "..." }
+ * // Invalid: {} or { url: "...", videoId: "..." }
+ */
+type ExactlyOne<T, Keys extends keyof T> = {
+    [K in Keys]: {
+        [P in K]-?: T[P];
+    } & {
+        [P in Exclude<Keys, K>]?: never;
+    };
+}[Keys] & Omit<T, Keys>;
+type TranscriptParams = {
     lang?: string;
     text?: boolean;
-}
-interface TranslateParams extends TranscriptParams {
+} & ExactlyOne<{
+    videoId: string;
+    url: string;
+}, 'videoId' | 'url'>;
+interface TranslateParams extends Omit<TranscriptParams, 'lang'> {
     lang: string;
 }
 declare class YouTubeService extends BaseClient {
+    /**
+     * Fetches a transcript for a YouTube video.
+     *
+     * @param params - Parameters for fetching the transcript
+     * @param params.videoId - The YouTube video ID (provide either this OR url)
+     * @param params.url - The YouTube video URL (provide either this OR videoId)
+     * @param params.lang - Optional language code for the transcript
+     * @param params.text - Optional flag to return plain text instead of timestamped list
+     * @returns A promise that resolves to the video transcript
+     */
     transcript(params: TranscriptParams): Promise<Transcript>;
+    /**
+     * Translates a YouTube video transcript to a specified language.
+     *
+     * @param params - Parameters for translating the transcript
+     * @param params.videoId - The YouTube video ID (provide either this OR url)
+     * @param params.url - The YouTube video URL (provide either this OR videoId)
+     * @param params.lang - The target language code for translation
+     * @param params.text - Optional flag to return plain text instead of timestamped list
+     * @returns A promise that resolves to the translated transcript
+     */
     translate(params: TranslateParams): Promise<TranslatedTranscript>;
 }
 
 declare class WebService extends BaseClient {
+    /**
+     * Extract content from any web page to Markdown format.
+     *
+     * @param url - URL of the webpage to scrape
+     * @returns A promise that resolves to the scraped content
+     */
     scrape(url: string): Promise<Scrape>;
+    /**
+     * Extract all links found on a webpage.
+     *
+     * @param url - URL of the webpage to map
+     * @returns A promise that resolves to a map of URLs found on the page
+     */
     map(url: string): Promise<Map>;
+    /**
+     * Create a crawl job to extract content from all pages on a website.
+     *
+     * @param request - Crawl request parameters
+     * @param request.url - URL of the website to crawl
+     * @param request.limit - Maximum number of pages to crawl (default: 100, max: 5000)
+     * @returns A promise that resolves to the crawl job id
+     */
+    crawl(request: CrawlRequest): Promise<Crawl>;
+    /**
+     * Get the status and results of a crawl job.
+     * Automatically handles pagination to retrieve all pages from the crawl.
+     *
+     * @param jobId - The ID of the crawl job to retrieve
+     * @returns A promise that resolves to the complete crawl job results
+     */
+    getCrawlResults(jobId: string): Promise<CrawlJob>;
 }
 
 declare class Supadata {
@@ -71,4 +148,4 @@ declare class Supadata {
     constructor(config: SupadataConfig);
 }
 
-export { BaseClient, type Map, type Scrape, Supadata, type SupadataConfig, SupadataError, type Transcript, type TranscriptChunk, type TranscriptParams, type TranslateParams, type TranslatedTranscript, WebService, YouTubeService };
+export { BaseClient, type Crawl, type CrawlJob, type CrawlRequest, type Map, type Scrape, Supadata, type SupadataConfig, SupadataError, type Transcript, type TranscriptChunk, type TranscriptParams, type TranslateParams, type TranslatedTranscript, WebService, YouTubeService };

package/dist/index.mjs

@@ -1,3 +1,3 @@
-var e=class extends Error{error;details;documentationUrl;constructor(r){super(r.message||"An unexpected error occurred"),this.error=r.error||"internal-error",this.details=r.details||"An unexpected error occurred",this.documentationUrl=r.documentationUrl||"",this.name="SupadataError";}};var c={403:{error:"invalid-request",message:"Invalid or missing API key",details:"Please ensure you have provided a valid API key"},404:{error:"invalid-request",message:"Endpoint does not exist",details:"The API endpoint you are trying to access does not exist"},429:{error:"limit-exceeded",message:"Limit exceeded",details:"You have exceeded the allowed request rate or quota limits"}},f=(t,r)=>t in c?new e({...c[t],message:c[t].message,details:r||c[t].details}):new e({error:"internal-error",message:"An unexpected error occurred",details:r});var s=class{config;constructor(r){this.config=r;}async fetch(r,m){let u=`${this.config.baseUrl||"https://api.supadata.ai/v1"}${r.startsWith("/")?r:`/${r}`}`;if(m){let a=new URLSearchParams;Object.entries(m).forEach(([T,l])=>{l!=null&&a.append(T,String(l));}),u+=`?${a.toString()}`;}let n=await fetch(u,{method:"GET",headers:{"x-api-key":this.config.apiKey,"Content-Type":"application/json"}}),g=n.headers.get("content-type");if(!n.ok){if([403,404,429].includes(n.status)){let a=await n.json();throw f(n.status,a.message)}if(g?.includes("application/json")){let a=await n.json();throw new e(a)}else throw new e({error:"internal-error",message:"Unexpected error response format",details:await n.text()})}try{if(!g?.includes("application/json"))throw new e({error:"internal-error",message:"Invalid response format",details:"Expected JSON response but received different content type"});return await n.json()}catch(a){throw new e({error:"internal-error",message:"Failed to parse response",details:a instanceof Error?a.message:"Unknown error"})}}};var p=class extends s{async transcript(r){return this.fetch("/youtube/transcript",r)}async translate(r){return this.fetch("/youtube/transcript/translate",r)}};var d=class extends s{async scrape(r){return this.fetch("/web/scrape",{url:r})}async map(r){return this.fetch("/web/map",{url:r})}};var h=class{youtube;web;constructor(r){this.youtube=new p(r),this.web=new d(r);}};
-export{s as BaseClient,h as Supadata,e as SupadataError,d as WebService,p as YouTubeService};//# sourceMappingURL=index.mjs.map
+var a=class extends Error{error;details;documentationUrl;constructor(e){super(e.message||"An unexpected error occurred"),this.error=e.error||"internal-error",this.details=e.details||"An unexpected error occurred",this.documentationUrl=e.documentationUrl||"",this.name="SupadataError";}};var u={403:{error:"invalid-request",message:"Invalid or missing API key",details:"Please ensure you have provided a valid API key"},404:{error:"invalid-request",message:"Endpoint does not exist",details:"The API endpoint you are trying to access does not exist"},429:{error:"limit-exceeded",message:"Limit exceeded",details:"You have exceeded the allowed request rate or quota limits"}},T=(n,e)=>n in u?new a({...u[n],message:u[n].message,details:e||u[n].details}):new a({error:"internal-error",message:"An unexpected error occurred",details:e});var c=class{config;constructor(e){this.config=e;}async fetch(e,r={},s="GET"){let t=`${this.config.baseUrl||"https://api.supadata.ai/v1"}${e.startsWith("/")?e:`/${e}`}`;if(s==="GET"&&Object.keys(r).length>0){let p=new URLSearchParams;Object.entries(r).forEach(([o,f])=>{f!=null&&p.append(o,String(f));}),t+=`?${p.toString()}`;}return this.fetchUrl(t,s,r)}async fetchUrl(e,r="GET",s){let i={method:r,headers:{"x-api-key":this.config.apiKey,"Content-Type":"application/json"}};r==="POST"&&s&&(i.body=JSON.stringify(s));let t=await fetch(e,i),p=t.headers.get("content-type");if(!t.ok){if([403,404,429].includes(t.status)){let o=await t.json();throw T(t.status,o.message)}if(p?.includes("application/json")){let o=await t.json();throw new a(o)}else throw new a({error:"internal-error",message:"Unexpected error response format",details:await t.text()})}try{if(!p?.includes("application/json"))throw new a({error:"internal-error",message:"Invalid response format",details:"Expected JSON response but received different content type"});return await t.json()}catch(o){throw new a({error:"internal-error",message:"Failed to parse response",details:o instanceof Error?o.message:"Unknown error"})}}};var g=class extends c{async transcript(e){return this.fetch("/youtube/transcript",e)}async translate(e){return this.fetch("/youtube/transcript/translate",e)}};var m=class extends c{async scrape(e){return this.fetch("/web/scrape",{url:e})}async map(e){return this.fetch("/web/map",{url:e})}async crawl(e){return this.fetch("/web/crawl",e,"POST")}async getCrawlResults(e){let r,s=[],i;do r=await(i?this.fetchUrl(i):this.fetch(`/web/crawl/${e}`)),r.pages&&(s=[...s,...r.pages]),i=r.next;while(i);return r}};var y=class{youtube;web;constructor(e){this.youtube=new g(e),this.web=new m(e);}};
+export{c as BaseClient,y as Supadata,a as SupadataError,m as WebService,g as YouTubeService};//# sourceMappingURL=index.mjs.map
 //# sourceMappingURL=index.mjs.map

package/dist/index.mjs.map

@@ -1 +1 @@
-{"version":3,"sources":["../src/types.ts","../src/gateway-error-mapper.ts","../src/client.ts","../src/services/youtube.ts","../src/services/web.ts","../src/index.ts"],"names":["SupadataError","error","GATEWAY_STATUS_ERRORS","mapGatewayError","statusCode","errorText","BaseClient","config","endpoint","params","url","queryParams","key","value","response","contentType","errorData","YouTubeService","WebService","Supadata"],"mappings":"AAqCaA,IAAAA,CAAAA,CAAN,cAA4B,KAAM,CACvC,MASA,OACA,CAAA,gBAAA,CAEA,YAAYC,CAKT,CAAA,CACD,MAAMA,CAAM,CAAA,OAAA,EAAW,8BAA8B,CACrD,CAAA,IAAA,CAAK,MAAQA,CAAM,CAAA,KAAA,EAAS,iBAC5B,IAAK,CAAA,OAAA,CAAUA,EAAM,OAAW,EAAA,8BAAA,CAChC,KAAK,gBAAmBA,CAAAA,CAAAA,CAAM,kBAAoB,EAClD,CAAA,IAAA,CAAK,KAAO,gBACd,CACF,ECvDA,IAAMC,CAAAA,CAAwB,CAC5B,GAAK,CAAA,CACH,MAAO,iBACP,CAAA,OAAA,CAAS,4BACT,CAAA,OAAA,CAAS,iDACX,CAAA,CACA,IAAK,CACH,KAAA,CAAO,kBACP,OAAS,CAAA,yBAAA,CACT,QAAS,0DACX,CAAA,CACA,IAAK,CACH,KAAA,CAAO,iBACP,OAAS,CAAA,gBAAA,CACT,QAAS,4DACX,CACF,EAEaC,CAAkB,CAAA,CAC7BC,CACAC,CAAAA,CAAAA,GAEID,CAAcF,IAAAA,CAAAA,CACT,IAAIF,CAAc,CAAA,CACvB,GAAGE,CACDE,CAAAA,CACF,EACA,OACEF,CAAAA,CAAAA,CAAsBE,CAAgD,CACnE,CAAA,OAAA,CACL,QACEC,CACAH,EAAAA,CAAAA,CAAsBE,CAAgD,CACnE,CAAA,OACP,CAAC,CAII,CAAA,IAAIJ,CAAc,CAAA,CACvB,KAAO,CAAA,gBAAA,CACP,QAAS,8BACT,CAAA,OAAA,CAASK,CACX,CAAC,CAAA,KC9CUC,CAAN,CAAA,KAAiB,CACZ,MAEV,CAAA,WAAA,CAAYC,EAAwB,CAClC,IAAA,CAAK,OAASA,EAChB,CAEA,MAAM,KACJC,CAAAA,CAAAA,CACAC,EACY,CAEZ,IAAIC,EAAM,CADM,EAAA,IAAA,CAAK,OAAO,OAAW,EAAA,4BACnB,GAClBF,CAAS,CAAA,UAAA,CAAW,GAAG,CAAIA,CAAAA,CAAAA,CAAW,IAAIA,CAAQ,CAAA,CACpD,GAEA,GAAIC,CAAAA,CAAQ,CACV,IAAME,CAAAA,CAAc,IAAI,eAAA,CACxB,MAAO,CAAA,OAAA,CAAQF,CAAM,CAAE,CAAA,OAAA,CAAQ,CAAC,CAACG,CAAAA,CAAKC,CAAK,CAAM,GAAA,CACpBA,GAAU,IACnCF,EAAAA,CAAAA,CAAY,OAAOC,CAAK,CAAA,MAAA,CAAOC,CAAK,CAAC,EAEzC,CAAC,CACDH,CAAAA,CAAAA,EAAO,CAAIC,CAAAA,EAAAA,CAAAA,CAAY,QAAS,EAAC,GACnC,CAEA,IAAMG,EAAW,MAAM,KAAA,CAAMJ,EAAK,CAChC,MAAA,CAAQ,MACR,OAAS,CAAA,CACP,YAAa,IAAK,CAAA,MAAA,CAAO,OACzB,cAAgB,CAAA,kBAClB,CACF,CAAC,CAAA,CAEKK,CAAcD,CAAAA,CAAAA,CAAS,OAAQ,CAAA,GAAA,CAAI,cAAc,CAEvD,CAAA,GAAI,CAACA,CAAS,CAAA,EAAA,CAAI,CAEhB,GAAI,CAAC,IAAK,GAAK,CAAA,GAAG,EAAE,QAASA,CAAAA,CAAAA,CAAS,MAAM,CAAG,CAAA,CAC7C,IAAME,CAAY,CAAA,MAAMF,EAAS,IAAK,EAAA,CACtC,MAAMX,CAAgBW,CAAAA,CAAAA,CAAS,OAAQE,CAAU,CAAA,OAAO,CAC1D,CAGA,GAAID,GAAa,QAAS,CAAA,kBAAkB,EAAG,CAC7C,IAAMC,EAAY,MAAMF,CAAAA,CAAS,MACjC,CAAA,MAAM,IAAId,CAAAA,CAAcgB,CAAS,CACnC,MAEQ,MAAA,IAAIhB,EAAc,CACtB,KAAA,CAAO,iBACP,OAAS,CAAA,kCAAA,CACT,QAAS,MAAMc,CAAAA,CAAS,MAC1B,CAAC,CAEL,CAEA,GAAI,CACF,GAAI,CAACC,CAAa,EAAA,QAAA,CAAS,kBAAkB,CAAA,CAC3C,MAAM,IAAIf,CAAAA,CAAc,CACtB,KAAO,CAAA,gBAAA,CACP,QAAS,yBACT,CAAA,OAAA,CAAS,4DACX,CAAC,CAAA,CAGH,OAAQ,MAAMc,CAAAA,CAAS,MACzB,CAAA,MAASb,EAAO,CACd,MAAM,IAAID,CAAAA,CAAc,CACtB,KAAA,CAAO,iBACP,OAAS,CAAA,0BAAA,CACT,QAASC,CAAiB,YAAA,KAAA,CAAQA,EAAM,OAAU,CAAA,eACpD,CAAC,CACH,CACF,CACF,ECjEO,IAAMgB,EAAN,cAA6BX,CAAW,CAC7C,MAAM,UAAA,CAAWG,EAA+C,CAC9D,OAAO,KAAK,KAAkB,CAAA,qBAAA,CAAuBA,CAAM,CAC7D,CAEA,MAAM,SAAUA,CAAAA,CAAAA,CAAwD,CACtE,OAAO,IAAA,CAAK,MACV,+BACAA,CAAAA,CACF,CACF,CACF,MCrBaS,CAAN,CAAA,cAAyBZ,CAAW,CACzC,MAAM,MAAA,CAAOI,EAA8B,CACzC,OAAO,KAAK,KAAc,CAAA,aAAA,CAAe,CAAE,GAAAA,CAAAA,CAAI,CAAC,CAClD,CAEA,MAAM,GAAIA,CAAAA,CAAAA,CAA2B,CACnC,OAAO,IAAA,CAAK,MAAW,UAAY,CAAA,CAAE,GAAAA,CAAAA,CAAI,CAAC,CAC5C,CACF,ECFO,IAAMS,EAAN,KAAe,CACX,QACA,GAET,CAAA,WAAA,CAAYZ,EAAwB,CAClC,IAAA,CAAK,QAAU,IAAIU,CAAAA,CAAeV,CAAM,CACxC,CAAA,IAAA,CAAK,IAAM,IAAIW,CAAAA,CAAWX,CAAM,EAClC,CACF","file":"index.mjs","sourcesContent":["export interface TranscriptChunk {\n  text: string;\n  offset: number;\n  duration: number;\n  lang: string;\n}\n\nexport interface Transcript {\n  content: TranscriptChunk[] | string;\n  lang: string;\n  availableLangs: string[];\n}\n\nexport interface TranslatedTranscript {\n  content: TranscriptChunk[] | string;\n  lang: string;\n}\n\nexport interface Scrape {\n  url: string;\n  content: string;\n  name: string;\n  description: string;\n  ogUrl: string;\n  countCharacters: number;\n  urls: string[];\n}\n\nexport interface Map {\n  urls: string[];\n}\n\nexport interface SupadataConfig {\n  apiKey: string;\n  baseUrl?: string;\n}\n\nexport class SupadataError extends Error {\n  error:\n    | 'invalid-request'\n    | 'missing-parameters'\n    | 'internal-error'\n    | 'transcript-unavailable'\n    | 'video-not-found'\n    | 'video-id-invalid'\n    | 'youtube-api-error'\n    | 'limit-exceeded';\n  details: string;\n  documentationUrl: string;\n\n  constructor(error: {\n    error: SupadataError['error'];\n    message?: string;\n    details?: string;\n    documentationUrl?: string;\n  }) {\n    super(error.message || 'An unexpected error occurred');\n    this.error = error.error || 'internal-error';\n    this.details = error.details || 'An unexpected error occurred';\n    this.documentationUrl = error.documentationUrl || '';\n    this.name = 'SupadataError';\n  }\n}\n","import { SupadataError } from './types';\n\n/*\nThe API gateway returns errors in text/plain content type. \nAs a temporary workaround we're mapping them to SupadataError.\n*/\n\nconst GATEWAY_STATUS_ERRORS = {\n  403: {\n    error: 'invalid-request' as const,\n    message: 'Invalid or missing API key',\n    details: 'Please ensure you have provided a valid API key',\n  },\n  404: {\n    error: 'invalid-request' as const,\n    message: 'Endpoint does not exist',\n    details: 'The API endpoint you are trying to access does not exist',\n  },\n  429: {\n    error: 'limit-exceeded' as const,\n    message: 'Limit exceeded',\n    details: 'You have exceeded the allowed request rate or quota limits',\n  },\n};\n\nexport const mapGatewayError = (\n  statusCode: number,\n  errorText: string\n): SupadataError => {\n  if (statusCode in GATEWAY_STATUS_ERRORS) {\n    return new SupadataError({\n      ...GATEWAY_STATUS_ERRORS[\n        statusCode as keyof typeof GATEWAY_STATUS_ERRORS\n      ],\n      message:\n        GATEWAY_STATUS_ERRORS[statusCode as keyof typeof GATEWAY_STATUS_ERRORS]\n          .message,\n      details:\n        errorText ||\n        GATEWAY_STATUS_ERRORS[statusCode as keyof typeof GATEWAY_STATUS_ERRORS]\n          .details,\n    });\n  }\n\n  // Default error if status code is not recognized\n  return new SupadataError({\n    error: 'internal-error',\n    message: 'An unexpected error occurred',\n    details: errorText,\n  });\n};\n","import { SupadataConfig, SupadataError } from './types.js';\nimport { mapGatewayError } from './gateway-error-mapper.js';\n\nexport class BaseClient {\n  protected config: SupadataConfig;\n\n  constructor(config: SupadataConfig) {\n    this.config = config;\n  }\n\n  async fetch<T>(\n    endpoint: string,\n    params: Record<string, unknown> | object\n  ): Promise<T> {\n    const baseUrl = this.config.baseUrl || 'https://api.supadata.ai/v1';\n    let url = `${baseUrl}${\n      endpoint.startsWith('/') ? endpoint : `/${endpoint}`\n    }`;\n\n    if (params) {\n      const queryParams = new URLSearchParams();\n      Object.entries(params).forEach(([key, value]) => {\n        if (value !== undefined && value !== null) {\n          queryParams.append(key, String(value));\n        }\n      });\n      url += `?${queryParams.toString()}`;\n    }\n\n    const response = await fetch(url, {\n      method: 'GET',\n      headers: {\n        'x-api-key': this.config.apiKey,\n        'Content-Type': 'application/json',\n      },\n    });\n\n    const contentType = response.headers.get('content-type');\n\n    if (!response.ok) {\n      // First check for gateway-specific status codes\n      if ([403, 404, 429].includes(response.status)) {\n        const errorData = await response.json();\n        throw mapGatewayError(response.status, errorData.message);\n      }\n\n      // Handle standard API errors\n      if (contentType?.includes('application/json')) {\n        const errorData = await response.json();\n        throw new SupadataError(errorData);\n      } else {\n        // Fallback for unexpected non-JSON errors\n        throw new SupadataError({\n          error: 'internal-error',\n          message: 'Unexpected error response format',\n          details: await response.text(),\n        });\n      }\n    }\n\n    try {\n      if (!contentType?.includes('application/json')) {\n        throw new SupadataError({\n          error: 'internal-error',\n          message: 'Invalid response format',\n          details: 'Expected JSON response but received different content type',\n        });\n      }\n\n      return (await response.json()) as T;\n    } catch (error) {\n      throw new SupadataError({\n        error: 'internal-error',\n        message: 'Failed to parse response',\n        details: error instanceof Error ? error.message : 'Unknown error',\n      });\n    }\n  }\n}\n","import { BaseClient } from '../client.js';\nimport { Transcript, TranslatedTranscript } from '../types.js';\n\nexport interface TranscriptParams {\n  videoId: string;\n  lang?: string;\n  text?: boolean;\n}\n\nexport interface TranslateParams extends TranscriptParams {\n  lang: string;\n}\n\nexport class YouTubeService extends BaseClient {\n  async transcript(params: TranscriptParams): Promise<Transcript> {\n    return this.fetch<Transcript>('/youtube/transcript', params);\n  }\n\n  async translate(params: TranslateParams): Promise<TranslatedTranscript> {\n    return this.fetch<TranslatedTranscript>(\n      '/youtube/transcript/translate',\n      params\n    );\n  }\n}\n","import { BaseClient } from '../client.js';\nimport { Scrape, Map } from '../types.js';\n\nexport class WebService extends BaseClient {\n  async scrape(url: string): Promise<Scrape> {\n    return this.fetch<Scrape>('/web/scrape', { url });\n  }\n\n  async map(url: string): Promise<Map> {\n    return this.fetch<Map>('/web/map', { url });\n  }\n}\n","import { SupadataConfig } from './types.js';\nimport { YouTubeService } from './services/youtube.js';\nimport { WebService } from './services/web.js';\n\nexport * from './types.js';\nexport * from './client.js';\nexport * from './services/youtube.js';\nexport * from './services/web.js';\n\nexport class Supadata {\n  readonly youtube: YouTubeService;\n  readonly web: WebService;\n\n  constructor(config: SupadataConfig) {\n    this.youtube = new YouTubeService(config);\n    this.web = new WebService(config);\n  }\n}\n"]}
+{"version":3,"sources":["../src/types.ts","../src/gateway-error-mapper.ts","../src/client.ts","../src/services/youtube.ts","../src/services/web.ts","../src/index.ts"],"names":["SupadataError","error","GATEWAY_STATUS_ERRORS","mapGatewayError","statusCode","errorText","BaseClient","config","endpoint","params","method","url","queryParams","key","value","body","options","response","contentType","errorData","YouTubeService","WebService","request","jobId","pages","nextUrl","Supadata"],"mappings":"AAoDaA,IAAAA,CAAAA,CAAN,cAA4B,KAAM,CACvC,KAAA,CASA,OACA,CAAA,gBAAA,CAEA,WAAYC,CAAAA,CAAAA,CAKT,CACD,KAAA,CAAMA,CAAM,CAAA,OAAA,EAAW,8BAA8B,CACrD,CAAA,IAAA,CAAK,KAAQA,CAAAA,CAAAA,CAAM,KAAS,EAAA,gBAAA,CAC5B,IAAK,CAAA,OAAA,CAAUA,CAAM,CAAA,OAAA,EAAW,8BAChC,CAAA,IAAA,CAAK,gBAAmBA,CAAAA,CAAAA,CAAM,kBAAoB,EAClD,CAAA,IAAA,CAAK,IAAO,CAAA,gBACd,CACF,ECtEA,IAAMC,CAAAA,CAAwB,CAC5B,GAAA,CAAK,CACH,KAAA,CAAO,iBACP,CAAA,OAAA,CAAS,6BACT,OAAS,CAAA,iDACX,CACA,CAAA,GAAA,CAAK,CACH,KAAA,CAAO,iBACP,CAAA,OAAA,CAAS,yBACT,CAAA,OAAA,CAAS,0DACX,CAAA,CACA,GAAK,CAAA,CACH,MAAO,gBACP,CAAA,OAAA,CAAS,gBACT,CAAA,OAAA,CAAS,4DACX,CACF,CAEaC,CAAAA,CAAAA,CAAkB,CAC7BC,CAAAA,CACAC,CAEID,GAAAA,CAAAA,IAAcF,CACT,CAAA,IAAIF,EAAc,CACvB,GAAGE,CACDE,CAAAA,CACF,CACA,CAAA,OAAA,CACEF,CAAsBE,CAAAA,CAAgD,CACnE,CAAA,OAAA,CACL,OACEC,CAAAA,CAAAA,EACAH,CAAsBE,CAAAA,CAAgD,EACnE,OACP,CAAC,CAII,CAAA,IAAIJ,CAAc,CAAA,CACvB,KAAO,CAAA,gBAAA,CACP,OAAS,CAAA,8BAAA,CACT,OAASK,CAAAA,CACX,CAAC,CAAA,CC9CUC,IAAAA,CAAAA,CAAN,KAAiB,CACZ,MAEV,CAAA,WAAA,CAAYC,CAAwB,CAAA,CAClC,IAAK,CAAA,MAAA,CAASA,EAChB,CAEA,MAAgB,KAAA,CACdC,CACAC,CAAAA,CAAAA,CAA8B,EAC9BC,CAAAA,CAAAA,CAAyB,KACb,CAAA,CAEZ,IAAIC,CAAAA,CAAM,CADM,EAAA,IAAA,CAAK,MAAO,CAAA,OAAA,EAAW,4BACnB,CAAA,EAClBH,CAAS,CAAA,UAAA,CAAW,GAAG,CAAIA,CAAAA,CAAAA,CAAW,CAAIA,CAAAA,EAAAA,CAAQ,CACpD,CAAA,CAAA,CAAA,CAEA,GAAIE,CAAAA,GAAW,KAAS,EAAA,MAAA,CAAO,IAAKD,CAAAA,CAAM,CAAE,CAAA,MAAA,CAAS,EAAG,CACtD,IAAMG,CAAc,CAAA,IAAI,eACxB,CAAA,MAAA,CAAO,OAAQH,CAAAA,CAAM,CAAE,CAAA,OAAA,CAAQ,CAAC,CAACI,CAAKC,CAAAA,CAAK,CAAM,GAAA,CACpBA,CAAU,EAAA,IAAA,EACnCF,CAAY,CAAA,MAAA,CAAOC,CAAK,CAAA,MAAA,CAAOC,CAAK,CAAC,EAEzC,CAAC,CACDH,CAAAA,CAAAA,EAAO,CAAIC,CAAAA,EAAAA,CAAAA,CAAY,UAAU,CAAA,EACnC,CAEA,OAAO,IAAK,CAAA,QAAA,CAAYD,CAAKD,CAAAA,CAAAA,CAAQD,CAAM,CAC7C,CAEA,MAAgB,QACdE,CAAAA,CAAAA,CACAD,EAAyB,KACzBK,CAAAA,CAAAA,CACY,CACZ,IAAMC,CAAuB,CAAA,CAC3B,MAAAN,CAAAA,CAAAA,CACA,OAAS,CAAA,CACP,WAAa,CAAA,IAAA,CAAK,MAAO,CAAA,MAAA,CACzB,eAAgB,kBAClB,CACF,CAEIA,CAAAA,CAAAA,GAAW,MAAUK,EAAAA,CAAAA,GACvBC,CAAQ,CAAA,IAAA,CAAO,IAAK,CAAA,SAAA,CAAUD,CAAI,CAAA,CAAA,CAGpC,IAAME,CAAAA,CAAW,MAAM,KAAA,CAAMN,CAAKK,CAAAA,CAAO,CAEnCE,CAAAA,CAAAA,CAAcD,CAAS,CAAA,OAAA,CAAQ,GAAI,CAAA,cAAc,CAEvD,CAAA,GAAI,CAACA,CAAAA,CAAS,EAAI,CAAA,CAEhB,GAAI,CAAC,GAAA,CAAK,GAAK,CAAA,GAAG,CAAE,CAAA,QAAA,CAASA,CAAS,CAAA,MAAM,CAAG,CAAA,CAC7C,IAAME,CAAAA,CAAY,MAAMF,CAAAA,CAAS,MACjC,CAAA,MAAMd,CAAgBc,CAAAA,CAAAA,CAAS,MAAQE,CAAAA,CAAAA,CAAU,OAAO,CAC1D,CAGA,GAAID,CAAa,EAAA,QAAA,CAAS,kBAAkB,CAAA,CAAG,CAC7C,IAAMC,CAAAA,CAAY,MAAMF,CAAAA,CAAS,IAAK,EAAA,CACtC,MAAM,IAAIjB,CAAcmB,CAAAA,CAAS,CACnC,CAAA,KAEQ,MAAA,IAAInB,CAAc,CAAA,CACtB,KAAO,CAAA,gBAAA,CACP,OAAS,CAAA,kCAAA,CACT,OAAS,CAAA,MAAMiB,CAAS,CAAA,IAAA,EAC1B,CAAC,CAEL,CAEA,GAAI,CACF,GAAI,CAACC,CAAAA,EAAa,QAAS,CAAA,kBAAkB,CAC3C,CAAA,MAAM,IAAIlB,CAAAA,CAAc,CACtB,KAAA,CAAO,gBACP,CAAA,OAAA,CAAS,yBACT,CAAA,OAAA,CAAS,4DACX,CAAC,CAAA,CAGH,OAAQ,MAAMiB,CAAS,CAAA,IAAA,EACzB,CAAA,MAAShB,CAAO,CAAA,CACd,MAAM,IAAID,CAAc,CAAA,CACtB,MAAO,gBACP,CAAA,OAAA,CAAS,0BACT,CAAA,OAAA,CAASC,CAAiB,YAAA,KAAA,CAAQA,CAAM,CAAA,OAAA,CAAU,eACpD,CAAC,CACH,CACF,CACF,ECtEamB,IAAAA,CAAAA,CAAN,cAA6Bd,CAAW,CAW7C,MAAM,UAAWG,CAAAA,CAAAA,CAA+C,CAC9D,OAAO,IAAK,CAAA,KAAA,CAAkB,qBAAuBA,CAAAA,CAAM,CAC7D,CAYA,MAAM,SAAUA,CAAAA,CAAAA,CAAwD,CACtE,OAAO,IAAK,CAAA,KAAA,CACV,+BACAA,CAAAA,CACF,CACF,CACF,ECnDO,IAAMY,CAAN,CAAA,cAAyBf,CAAW,CAOzC,MAAM,MAAOK,CAAAA,CAAAA,CAA8B,CACzC,OAAO,IAAK,CAAA,KAAA,CAAc,aAAe,CAAA,CAAE,GAAAA,CAAAA,CAAI,CAAC,CAClD,CAQA,MAAM,GAAA,CAAIA,CAA2B,CAAA,CACnC,OAAO,IAAA,CAAK,KAAW,CAAA,UAAA,CAAY,CAAE,GAAA,CAAAA,CAAI,CAAC,CAC5C,CAUA,MAAM,KAAA,CAAMW,CAAuC,CAAA,CACjD,OAAO,IAAA,CAAK,KAAa,CAAA,YAAA,CAAcA,CAAS,CAAA,MAAM,CACxD,CASA,MAAM,eAAA,CAAgBC,CAAkC,CAAA,CACtD,IAAIN,CACAO,CAAAA,CAAAA,CAAkB,EAAC,CACnBC,CAEJ,CAAA,GACER,CAAW,CAAA,MAAOQ,CACd,CAAA,IAAA,CAAK,QAAmBA,CAAAA,CAAO,CAC/B,CAAA,IAAA,CAAK,MAAgB,CAAcF,WAAAA,EAAAA,CAAK,CAAE,CAAA,CAAA,CAAA,CAE1CN,CAAS,CAAA,KAAA,GACXO,CAAQ,CAAA,CAAC,GAAGA,CAAAA,CAAO,GAAGP,CAAAA,CAAS,KAAK,CAAA,CAAA,CAEtCQ,EAAUR,CAAS,CAAA,IAAA,CAAA,MACZQ,CAET,EAAA,OAAOR,CACT,CACF,ECpDO,IAAMS,CAAN,CAAA,KAAe,CACX,OAAA,CACA,GAET,CAAA,WAAA,CAAYnB,CAAwB,CAAA,CAClC,IAAK,CAAA,OAAA,CAAU,IAAIa,CAAAA,CAAeb,CAAM,CAAA,CACxC,IAAK,CAAA,GAAA,CAAM,IAAIc,CAAAA,CAAWd,CAAM,EAClC,CACF","file":"index.mjs","sourcesContent":["export interface TranscriptChunk {\n  text: string;\n  offset: number;\n  duration: number;\n  lang: string;\n}\n\nexport interface Transcript {\n  content: TranscriptChunk[] | string;\n  lang: string;\n  availableLangs: string[];\n}\n\nexport interface TranslatedTranscript {\n  content: TranscriptChunk[] | string;\n  lang: string;\n}\n\nexport interface Scrape {\n  url: string;\n  content: string;\n  name: string;\n  description: string;\n  ogUrl: string;\n  countCharacters: number;\n  urls: string[];\n}\n\nexport interface Map {\n  urls: string[];\n}\n\nexport interface CrawlRequest {\n  url: string;\n  limit?: number;\n}\n\nexport interface Crawl {\n  jobId: string;\n}\n\nexport interface CrawlJob {\n  status: 'scraping' | 'completed' | 'failed' | 'cancelled';\n  pages?: Scrape[];\n  next?: string;\n}\n\nexport interface SupadataConfig {\n  apiKey: string;\n  baseUrl?: string;\n}\n\nexport class SupadataError extends Error {\n  error:\n    | 'invalid-request'\n    | 'missing-parameters'\n    | 'internal-error'\n    | 'transcript-unavailable'\n    | 'video-not-found'\n    | 'video-id-invalid'\n    | 'youtube-api-error'\n    | 'limit-exceeded';\n  details: string;\n  documentationUrl: string;\n\n  constructor(error: {\n    error: SupadataError['error'];\n    message?: string;\n    details?: string;\n    documentationUrl?: string;\n  }) {\n    super(error.message || 'An unexpected error occurred');\n    this.error = error.error || 'internal-error';\n    this.details = error.details || 'An unexpected error occurred';\n    this.documentationUrl = error.documentationUrl || '';\n    this.name = 'SupadataError';\n  }\n}\n","import { SupadataError } from './types';\n\n/*\nThe API gateway returns errors in text/plain content type. \nAs a temporary workaround we're mapping them to SupadataError.\n*/\n\nconst GATEWAY_STATUS_ERRORS = {\n  403: {\n    error: 'invalid-request' as const,\n    message: 'Invalid or missing API key',\n    details: 'Please ensure you have provided a valid API key',\n  },\n  404: {\n    error: 'invalid-request' as const,\n    message: 'Endpoint does not exist',\n    details: 'The API endpoint you are trying to access does not exist',\n  },\n  429: {\n    error: 'limit-exceeded' as const,\n    message: 'Limit exceeded',\n    details: 'You have exceeded the allowed request rate or quota limits',\n  },\n};\n\nexport const mapGatewayError = (\n  statusCode: number,\n  errorText: string\n): SupadataError => {\n  if (statusCode in GATEWAY_STATUS_ERRORS) {\n    return new SupadataError({\n      ...GATEWAY_STATUS_ERRORS[\n        statusCode as keyof typeof GATEWAY_STATUS_ERRORS\n      ],\n      message:\n        GATEWAY_STATUS_ERRORS[statusCode as keyof typeof GATEWAY_STATUS_ERRORS]\n          .message,\n      details:\n        errorText ||\n        GATEWAY_STATUS_ERRORS[statusCode as keyof typeof GATEWAY_STATUS_ERRORS]\n          .details,\n    });\n  }\n\n  // Default error if status code is not recognized\n  return new SupadataError({\n    error: 'internal-error',\n    message: 'An unexpected error occurred',\n    details: errorText,\n  });\n};\n","import { SupadataConfig, SupadataError } from './types.js';\nimport { mapGatewayError } from './gateway-error-mapper.js';\n\nexport class BaseClient {\n  protected config: SupadataConfig;\n\n  constructor(config: SupadataConfig) {\n    this.config = config;\n  }\n\n  protected async fetch<T>(\n    endpoint: string,\n    params: Record<string, any> = {},\n    method: 'GET' | 'POST' = 'GET'\n  ): Promise<T> {\n    const baseUrl = this.config.baseUrl || 'https://api.supadata.ai/v1';\n    let url = `${baseUrl}${\n      endpoint.startsWith('/') ? endpoint : `/${endpoint}`\n    }`;\n\n    if (method === 'GET' && Object.keys(params).length > 0) {\n      const queryParams = new URLSearchParams();\n      Object.entries(params).forEach(([key, value]) => {\n        if (value !== undefined && value !== null) {\n          queryParams.append(key, String(value));\n        }\n      });\n      url += `?${queryParams.toString()}`;\n    }\n\n    return this.fetchUrl<T>(url, method, params);\n  }\n\n  protected async fetchUrl<T>(\n    url: string,\n    method: 'GET' | 'POST' = 'GET',\n    body?: Record<string, any>\n  ): Promise<T> {\n    const options: RequestInit = {\n      method,\n      headers: {\n        'x-api-key': this.config.apiKey,\n        'Content-Type': 'application/json',\n      },\n    };\n\n    if (method === 'POST' && body) {\n      options.body = JSON.stringify(body);\n    }\n\n    const response = await fetch(url, options);\n\n    const contentType = response.headers.get('content-type');\n\n    if (!response.ok) {\n      // First check for gateway-specific status codes\n      if ([403, 404, 429].includes(response.status)) {\n        const errorData = await response.json();\n        throw mapGatewayError(response.status, errorData.message);\n      }\n\n      // Handle standard API errors\n      if (contentType?.includes('application/json')) {\n        const errorData = await response.json();\n        throw new SupadataError(errorData);\n      } else {\n        // Fallback for unexpected non-JSON errors\n        throw new SupadataError({\n          error: 'internal-error',\n          message: 'Unexpected error response format',\n          details: await response.text(),\n        });\n      }\n    }\n\n    try {\n      if (!contentType?.includes('application/json')) {\n        throw new SupadataError({\n          error: 'internal-error',\n          message: 'Invalid response format',\n          details: 'Expected JSON response but received different content type',\n        });\n      }\n\n      return (await response.json()) as T;\n    } catch (error) {\n      throw new SupadataError({\n        error: 'internal-error',\n        message: 'Failed to parse response',\n        details: error instanceof Error ? error.message : 'Unknown error',\n      });\n    }\n  }\n}\n","import { BaseClient } from '../client.js';\nimport { Transcript, TranslatedTranscript } from '../types.js';\n\n/**\n * Ensures exactly one property from the specified keys is provided.\n * @example\n * // Valid: { url: \"...\" } or { videoId: \"...\" }\n * // Invalid: {} or { url: \"...\", videoId: \"...\" }\n */\ntype ExactlyOne<T, Keys extends keyof T> = {\n  [K in Keys]: { [P in K]-?: T[P] } & { [P in Exclude<Keys, K>]?: never };\n}[Keys] &\n  Omit<T, Keys>;\n\nexport type TranscriptParams = {\n  lang?: string;\n  text?: boolean;\n} & ExactlyOne<{ videoId: string; url: string }, 'videoId' | 'url'>;\n\nexport interface TranslateParams extends Omit<TranscriptParams, 'lang'> {\n  lang: string;\n}\n\nexport class YouTubeService extends BaseClient {\n  /**\n   * Fetches a transcript for a YouTube video.\n   *\n   * @param params - Parameters for fetching the transcript\n   * @param params.videoId - The YouTube video ID (provide either this OR url)\n   * @param params.url - The YouTube video URL (provide either this OR videoId)\n   * @param params.lang - Optional language code for the transcript\n   * @param params.text - Optional flag to return plain text instead of timestamped list\n   * @returns A promise that resolves to the video transcript\n   */\n  async transcript(params: TranscriptParams): Promise<Transcript> {\n    return this.fetch<Transcript>('/youtube/transcript', params);\n  }\n\n  /**\n   * Translates a YouTube video transcript to a specified language.\n   *\n   * @param params - Parameters for translating the transcript\n   * @param params.videoId - The YouTube video ID (provide either this OR url)\n   * @param params.url - The YouTube video URL (provide either this OR videoId)\n   * @param params.lang - The target language code for translation\n   * @param params.text - Optional flag to return plain text instead of timestamped list\n   * @returns A promise that resolves to the translated transcript\n   */\n  async translate(params: TranslateParams): Promise<TranslatedTranscript> {\n    return this.fetch<TranslatedTranscript>(\n      '/youtube/transcript/translate',\n      params\n    );\n  }\n}\n","import { BaseClient } from '../client.js';\nimport { Scrape, Map, CrawlRequest, Crawl, CrawlJob } from '../types.js';\n\nexport class WebService extends BaseClient {\n  /**\n   * Extract content from any web page to Markdown format.\n   *\n   * @param url - URL of the webpage to scrape\n   * @returns A promise that resolves to the scraped content\n   */\n  async scrape(url: string): Promise<Scrape> {\n    return this.fetch<Scrape>('/web/scrape', { url });\n  }\n\n  /**\n   * Extract all links found on a webpage.\n   *\n   * @param url - URL of the webpage to map\n   * @returns A promise that resolves to a map of URLs found on the page\n   */\n  async map(url: string): Promise<Map> {\n    return this.fetch<Map>('/web/map', { url });\n  }\n\n  /**\n   * Create a crawl job to extract content from all pages on a website.\n   *\n   * @param request - Crawl request parameters\n   * @param request.url - URL of the website to crawl\n   * @param request.limit - Maximum number of pages to crawl (default: 100, max: 5000)\n   * @returns A promise that resolves to the crawl job id\n   */\n  async crawl(request: CrawlRequest): Promise<Crawl> {\n    return this.fetch<Crawl>('/web/crawl', request, 'POST');\n  }\n\n  /**\n   * Get the status and results of a crawl job.\n   * Automatically handles pagination to retrieve all pages from the crawl.\n   *\n   * @param jobId - The ID of the crawl job to retrieve\n   * @returns A promise that resolves to the complete crawl job results\n   */\n  async getCrawlResults(jobId: string): Promise<CrawlJob> {\n    let response: CrawlJob;\n    let pages: Scrape[] = [];\n    let nextUrl: string | undefined;\n\n    do {\n      response = await (nextUrl\n        ? this.fetchUrl<CrawlJob>(nextUrl)\n        : this.fetch<CrawlJob>(`/web/crawl/${jobId}`));\n\n      if (response.pages) {\n        pages = [...pages, ...response.pages];\n      }\n      nextUrl = response.next;\n    } while (nextUrl);\n\n    return response;\n  }\n}\n","import { SupadataConfig } from './types.js';\nimport { YouTubeService } from './services/youtube.js';\nimport { WebService } from './services/web.js';\n\nexport * from './types.js';\nexport * from './client.js';\nexport * from './services/youtube.js';\nexport * from './services/web.js';\n\nexport class Supadata {\n  readonly youtube: YouTubeService;\n  readonly web: WebService;\n\n  constructor(config: SupadataConfig) {\n    this.youtube = new YouTubeService(config);\n    this.web = new WebService(config);\n  }\n}\n"]}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@supadata/js",
-  "version": "1.0.2",
+  "version": "1.0.4",
   "description": "TypeScript / JavaScript SDK for Supadata API",
   "homepage": "https://supadata.ai",
   "repository": "https://github.com/supadata-ai/js",