@jongleberry/vurst-html 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/index.d.ts +68 -0
- package/index.js +31 -0
- package/package.json +24 -0
package/index.d.ts
ADDED
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
/// <reference types="node" />
|
|
2
|
+
|
|
3
|
+
export interface NapiSanitizeRssHtmlOptions {
|
|
4
|
+
proxyImages?: boolean
|
|
5
|
+
imageProxyUrlPrefix?: string
|
|
6
|
+
imageProxySigningKeys?: Array<string>
|
|
7
|
+
}
|
|
8
|
+
|
|
9
|
+
export interface SanitizeRssHtmlResult {
|
|
10
|
+
html: Buffer
|
|
11
|
+
firstImageSrc?: string
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
export interface ExtractDomRemovalsOptions {
|
|
15
|
+
boilerplatePatterns?: Array<string>
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
export interface ExtractDomRemovalsResult {
|
|
19
|
+
cssSelectorsToRemove: Array<string>
|
|
20
|
+
htmlToRemove: Array<string>
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
export interface CrawlerHtmlToMarkdownOptions {
|
|
24
|
+
cssSelectorsToRemove?: Array<string>
|
|
25
|
+
contentSelectors?: Array<string>
|
|
26
|
+
linkTextContentToRemove?: Array<string>
|
|
27
|
+
linkHrefsToRemove?: Array<string>
|
|
28
|
+
linkRelTokensToRemove?: Array<string>
|
|
29
|
+
useTextDensityFilter?: boolean
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
export interface CrawlerHtmlToMarkdownResult {
|
|
33
|
+
title?: string
|
|
34
|
+
meta: Record<string, unknown>
|
|
35
|
+
links: Record<string, unknown>
|
|
36
|
+
content: string
|
|
37
|
+
canonicalUrl?: string
|
|
38
|
+
lang?: string
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
export declare function sanitizeRssHtml(
|
|
42
|
+
html: Buffer,
|
|
43
|
+
options?: NapiSanitizeRssHtmlOptions,
|
|
44
|
+
): Promise<SanitizeRssHtmlResult>
|
|
45
|
+
|
|
46
|
+
export declare function sanitizeRssHtmlBatch(
|
|
47
|
+
inputs: Array<Buffer>,
|
|
48
|
+
options?: NapiSanitizeRssHtmlOptions,
|
|
49
|
+
): Promise<Array<SanitizeRssHtmlResult>>
|
|
50
|
+
|
|
51
|
+
export declare function htmlToEmbeddingText(html: Buffer): Promise<string>
|
|
52
|
+
|
|
53
|
+
export declare function extractDomRemovals(
|
|
54
|
+
htmlPages: Array<Buffer>,
|
|
55
|
+
options?: ExtractDomRemovalsOptions,
|
|
56
|
+
): Promise<ExtractDomRemovalsResult>
|
|
57
|
+
|
|
58
|
+
export declare function applyDomRemovalsToHtml(
|
|
59
|
+
html: Buffer,
|
|
60
|
+
removals: ExtractDomRemovalsResult,
|
|
61
|
+
): Promise<Buffer>
|
|
62
|
+
|
|
63
|
+
export declare function getContentFromHtml(
|
|
64
|
+
htmlBuffer: Buffer,
|
|
65
|
+
options: CrawlerHtmlToMarkdownOptions,
|
|
66
|
+
): Promise<CrawlerHtmlToMarkdownResult>
|
|
67
|
+
|
|
68
|
+
export declare function sanitizePromptInjection(content: Buffer, isTitle?: boolean): Promise<Buffer>
|
package/index.js
ADDED
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
'use strict'
|
|
2
|
+
|
|
3
|
+
// Platform dispatcher for @jongleberry/vurst-html.
|
|
4
|
+
|
|
5
|
+
const { platform, arch } = process
|
|
6
|
+
|
|
7
|
+
function loadBinding() {
|
|
8
|
+
switch (`${platform}-${arch}`) {
|
|
9
|
+
case 'darwin-arm64':
|
|
10
|
+
return require('./vurst-html.darwin-arm64.node')
|
|
11
|
+
case 'linux-x64':
|
|
12
|
+
return require('./vurst-html.linux-x64-gnu.node')
|
|
13
|
+
case 'linux-arm64':
|
|
14
|
+
return require('./vurst-html.linux-arm64-gnu.node')
|
|
15
|
+
default:
|
|
16
|
+
throw new Error(
|
|
17
|
+
`Unsupported platform: ${platform}-${arch}. ` +
|
|
18
|
+
`@jongleberry/vurst-html supports darwin-arm64, linux-x64 (glibc), and linux-arm64 (glibc).`,
|
|
19
|
+
)
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
const binding = loadBinding()
|
|
24
|
+
|
|
25
|
+
module.exports.sanitizeRssHtml = binding.sanitizeRssHtml
|
|
26
|
+
module.exports.sanitizeRssHtmlBatch = binding.sanitizeRssHtmlBatch
|
|
27
|
+
module.exports.htmlToEmbeddingText = binding.htmlToEmbeddingText
|
|
28
|
+
module.exports.extractDomRemovals = binding.extractDomRemovals
|
|
29
|
+
module.exports.applyDomRemovalsToHtml = binding.applyDomRemovalsToHtml
|
|
30
|
+
module.exports.getContentFromHtml = binding.getContentFromHtml
|
|
31
|
+
module.exports.sanitizePromptInjection = binding.sanitizePromptInjection
|
package/package.json
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@jongleberry/vurst-html",
|
|
3
|
+
"version": "0.0.1",
|
|
4
|
+
"description": "Rust + N-API: HTML sanitization, extraction, and boilerstrip.",
|
|
5
|
+
"main": "index.js",
|
|
6
|
+
"types": "index.d.ts",
|
|
7
|
+
"license": "MIT",
|
|
8
|
+
"engines": { "node": ">= 18" },
|
|
9
|
+
"repository": { "type": "git", "url": "git+https://github.com/jonathanong/vurst.git" },
|
|
10
|
+
"keywords": ["rust", "napi", "html", "sanitize", "embedding", "boilerstrip"],
|
|
11
|
+
"files": ["index.js", "index.d.ts", "README.md", "*.node"],
|
|
12
|
+
"napi": {
|
|
13
|
+
"binaryName": "vurst-html",
|
|
14
|
+
"targets": ["aarch64-apple-darwin", "aarch64-unknown-linux-gnu", "x86_64-unknown-linux-gnu"]
|
|
15
|
+
},
|
|
16
|
+
"scripts": {
|
|
17
|
+
"build": "napi build --platform --release",
|
|
18
|
+
"build:debug": "napi build --platform",
|
|
19
|
+
"test": "node --test __test__/*.test.mjs",
|
|
20
|
+
"prepublishOnly": "napi prepublish -t npm"
|
|
21
|
+
},
|
|
22
|
+
"publishConfig": { "access": "public", "provenance": true },
|
|
23
|
+
"devDependencies": { "@napi-rs/cli": "^3.6.2", "@types/node": "^24" }
|
|
24
|
+
}
|