@kreuzberg/kreuzcrawl 0.1.0-rc.10

package/Cargo.toml

@@ -0,0 +1,18 @@
+[package]
+name = "kreuzcrawl-node"
+version = "0.1.0-rc.10"
+edition = "2024"
+license = "Elastic-2.0"
+
+[lib]
+crate-type = ["cdylib"]
+
+[dependencies]
+kreuzcrawl = { path = "../kreuzcrawl" }
+napi = { version = "3", features = ["async"] }
+napi-derive = "3"
+serde = { version = "1", features = ["derive"] }
+serde_json = "1"
+
+[build-dependencies]
+napi-build = "2"

package/build.rs

@@ -0,0 +1,5 @@
+extern crate napi_build;
+
+fn main() {
+    napi_build::setup();
+}

package/index.d.ts

@@ -0,0 +1,404 @@
+/* auto-generated by NAPI-RS */
+/* eslint-disable */
+export declare class JsCrawlEngineHandle {
+
+}
+
+export declare function batchCrawl(engine: JsCrawlEngineHandle, urls: Array<string>): Promise<Array<JsBatchCrawlResult>>
+
+export declare function batchScrape(engine: JsCrawlEngineHandle, urls: Array<string>): Promise<Array<JsBatchScrapeResult>>
+
+export declare function crawl(engine: JsCrawlEngineHandle, url: string): Promise<JsCrawlResult>
+
+export declare function createEngine(config?: JsCrawlConfig | undefined | null): JsCrawlEngineHandle
+
+export interface JsActionResult {
+  actionIndex?: number
+  actionType?: string
+  success?: boolean
+  data?: string
+  error?: string
+}
+
+export interface JsArticleMetadata {
+  publishedTime?: string
+  modifiedTime?: string
+  author?: string
+  section?: string
+  tags?: Array<string>
+}
+
+export declare const enum JsAssetCategory {
+  Document = 'document',
+  Image = 'image',
+  Audio = 'audio',
+  Video = 'video',
+  Font = 'font',
+  Stylesheet = 'stylesheet',
+  Script = 'script',
+  Archive = 'archive',
+  Data = 'data',
+  Other = 'other'
+}
+
+export interface JsAuthConfig {
+  type: string
+  username?: string
+  password?: string
+  token?: string
+  name?: string
+  value?: string
+}
+
+export interface JsBatchCrawlResult {
+  url?: string
+  result?: JsCrawlResult
+  error?: string
+}
+
+export interface JsBatchScrapeResult {
+  url?: string
+  result?: JsScrapeResult
+  error?: string
+}
+
+export interface JsBrowserConfig {
+  mode?: JsBrowserMode
+  endpoint?: string
+  timeout?: number
+  wait?: JsBrowserWait
+  waitSelector?: string
+  extraWait?: number
+}
+
+export declare const enum JsBrowserMode {
+  Auto = 'auto',
+  Always = 'always',
+  Never = 'never'
+}
+
+export declare const enum JsBrowserWait {
+  NetworkIdle = 'network_idle',
+  Selector = 'selector',
+  Fixed = 'fixed'
+}
+
+export interface JsCachedPage {
+  url?: string
+  statusCode?: number
+  contentType?: string
+  body?: string
+  etag?: string
+  lastModified?: string
+  cachedAt?: number
+}
+
+export interface JsCitationReference {
+  index?: number
+  url?: string
+  text?: string
+}
+
+export interface JsCitationResult {
+  content?: string
+  references?: Array<JsCitationReference>
+}
+
+export interface JsCookieInfo {
+  name?: string
+  value?: string
+  domain?: string
+  path?: string
+}
+
+export interface JsCrawlConfig {
+  maxDepth?: number
+  maxPages?: number
+  maxConcurrent?: number
+  respectRobotsTxt?: boolean
+  userAgent?: string
+  stayOnDomain?: boolean
+  allowSubdomains?: boolean
+  includePaths?: Array<string>
+  excludePaths?: Array<string>
+  customHeaders?: Record<string, string>
+  requestTimeout?: number
+  maxRedirects?: number
+  retryCount?: number
+  retryCodes?: Array<number>
+  cookiesEnabled?: boolean
+  auth?: JsAuthConfig
+  maxBodySize?: number
+  mainContentOnly?: boolean
+  removeTags?: Array<string>
+  mapLimit?: number
+  mapSearch?: string
+  downloadAssets?: boolean
+  assetTypes?: Array<JsAssetCategory>
+  maxAssetSize?: number
+  browser?: JsBrowserConfig
+  proxy?: JsProxyConfig
+  userAgents?: Array<string>
+  captureScreenshot?: boolean
+  downloadDocuments?: boolean
+  documentMaxSize?: number
+  documentMimeTypes?: Array<string>
+  warcOutput?: string
+  browserProfile?: string
+  saveBrowserProfile?: boolean
+}
+
+export declare const enum JsCrawlEvent {
+  Page = 'Page',
+  Error = 'Error',
+  Complete = 'Complete'
+}
+
+export interface JsCrawlPageResult {
+  url?: string
+  normalizedUrl?: string
+  statusCode?: number
+  contentType?: string
+  html?: string
+  bodySize?: number
+  metadata?: JsPageMetadata
+  links?: Array<JsLinkInfo>
+  images?: Array<JsImageInfo>
+  feeds?: Array<JsFeedInfo>
+  jsonLd?: Array<JsJsonLdEntry>
+  depth?: number
+  stayedOnDomain?: boolean
+  wasSkipped?: boolean
+  isPdf?: boolean
+  detectedCharset?: string
+  markdown?: JsMarkdownResult
+  extractedData?: string
+  extractionMeta?: JsExtractionMeta
+  downloadedDocument?: JsDownloadedDocument
+}
+
+export interface JsCrawlResult {
+  pages?: Array<JsCrawlPageResult>
+  finalUrl?: string
+  redirectCount?: number
+  wasSkipped?: boolean
+  error?: string
+  cookies?: Array<JsCookieInfo>
+  normalizedUrls?: Array<string>
+}
+
+export interface JsDownloadedAsset {
+  url?: string
+  contentHash?: string
+  mimeType?: string
+  size?: number
+  assetCategory?: JsAssetCategory
+  htmlTag?: string
+}
+
+export interface JsDownloadedDocument {
+  url?: string
+  mimeType?: string
+  content?: Array<number>
+  size?: number
+  filename?: string
+  contentHash?: string
+  headers?: Record<string, string>
+}
+
+export interface JsExtractionMeta {
+  cost?: number
+  promptTokens?: number
+  completionTokens?: number
+  model?: string
+  chunksProcessed?: number
+}
+
+export interface JsFaviconInfo {
+  url?: string
+  rel?: string
+  sizes?: string
+  mimeType?: string
+}
+
+export interface JsFeedInfo {
+  url?: string
+  title?: string
+  feedType?: JsFeedType
+}
+
+export declare const enum JsFeedType {
+  Rss = 'rss',
+  Atom = 'atom',
+  JsonFeed = 'json_feed'
+}
+
+export interface JsHeadingInfo {
+  level?: number
+  text?: string
+}
+
+export interface JsHreflangEntry {
+  lang?: string
+  url?: string
+}
+
+export interface JsImageInfo {
+  url?: string
+  alt?: string
+  width?: number
+  height?: number
+  source?: JsImageSource
+}
+
+export declare const enum JsImageSource {
+  Img = 'img',
+  PictureSource = 'picture_source',
+  OgImage = 'og_image',
+  TwitterImage = 'twitter_image'
+}
+
+export interface JsInteractionResult {
+  actionResults?: Array<JsActionResult>
+  finalHtml?: string
+  finalUrl?: string
+  screenshot?: Array<number>
+}
+
+export interface JsJsonLdEntry {
+  schemaType?: string
+  name?: string
+  raw?: string
+}
+
+export interface JsLinkInfo {
+  url?: string
+  text?: string
+  linkType?: JsLinkType
+  rel?: string
+  nofollow?: boolean
+}
+
+export declare const enum JsLinkType {
+  Internal = 'internal',
+  External = 'external',
+  Anchor = 'anchor',
+  Document = 'document'
+}
+
+export interface JsMapResult {
+  urls?: Array<JsSitemapUrl>
+}
+
+export interface JsMarkdownResult {
+  content?: string
+  documentStructure?: string
+  tables?: Array<string>
+  warnings?: Array<string>
+  citations?: JsCitationResult
+  fitContent?: string
+}
+
+export interface JsPageMetadata {
+  title?: string
+  description?: string
+  canonicalUrl?: string
+  keywords?: string
+  author?: string
+  viewport?: string
+  themeColor?: string
+  generator?: string
+  robots?: string
+  htmlLang?: string
+  htmlDir?: string
+  ogTitle?: string
+  ogType?: string
+  ogImage?: string
+  ogDescription?: string
+  ogUrl?: string
+  ogSiteName?: string
+  ogLocale?: string
+  ogVideo?: string
+  ogAudio?: string
+  ogLocaleAlternates?: Array<string>
+  twitterCard?: string
+  twitterTitle?: string
+  twitterDescription?: string
+  twitterImage?: string
+  twitterSite?: string
+  twitterCreator?: string
+  dcTitle?: string
+  dcCreator?: string
+  dcSubject?: string
+  dcDescription?: string
+  dcPublisher?: string
+  dcDate?: string
+  dcType?: string
+  dcFormat?: string
+  dcIdentifier?: string
+  dcLanguage?: string
+  dcRights?: string
+  article?: JsArticleMetadata
+  hreflangs?: Array<JsHreflangEntry>
+  favicons?: Array<JsFaviconInfo>
+  headings?: Array<JsHeadingInfo>
+  wordCount?: number
+}
+
+export interface JsProxyConfig {
+  url?: string
+  username?: string
+  password?: string
+}
+
+export interface JsResponseMeta {
+  etag?: string
+  lastModified?: string
+  cacheControl?: string
+  server?: string
+  xPoweredBy?: string
+  contentLanguage?: string
+  contentEncoding?: string
+}
+
+export interface JsScrapeResult {
+  statusCode?: number
+  contentType?: string
+  html?: string
+  bodySize?: number
+  metadata?: JsPageMetadata
+  links?: Array<JsLinkInfo>
+  images?: Array<JsImageInfo>
+  feeds?: Array<JsFeedInfo>
+  jsonLd?: Array<JsJsonLdEntry>
+  isAllowed?: boolean
+  crawlDelay?: number
+  noindexDetected?: boolean
+  nofollowDetected?: boolean
+  xRobotsTag?: string
+  isPdf?: boolean
+  wasSkipped?: boolean
+  detectedCharset?: string
+  mainContentOnly?: boolean
+  authHeaderSent?: boolean
+  responseMeta?: JsResponseMeta
+  assets?: Array<JsDownloadedAsset>
+  jsRenderHint?: boolean
+  browserUsed?: boolean
+  markdown?: JsMarkdownResult
+  extractedData?: string
+  extractionMeta?: JsExtractionMeta
+  screenshot?: Array<number>
+  downloadedDocument?: JsDownloadedDocument
+}
+
+export interface JsSitemapUrl {
+  url?: string
+  lastmod?: string
+  changefreq?: string
+  priority?: string
+}
+
+export declare function mapUrls(engine: JsCrawlEngineHandle, url: string): Promise<JsMapResult>
+
+export declare function scrape(engine: JsCrawlEngineHandle, url: string): Promise<JsScrapeResult>