@framers/agentos-ext-content-extraction 0.1.1 → 0.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +81 -0
- package/manifest.json +1 -1
- package/package.json +3 -2
package/README.md
ADDED
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
# Content extraction
|
|
2
|
+
|
|
3
|
+
This pack gives an agent tools to extract web content, YouTube transcripts, Wikipedia articles, PDF text and structured page data. It uses `axios` for HTTP requests, YouTube oEmbed and caption endpoints, and Wikipedia REST and MediaWiki APIs.
|
|
4
|
+
|
|
5
|
+
## Installation
|
|
6
|
+
|
|
7
|
+
```bash
|
|
8
|
+
npm install @framers/agentos-ext-content-extraction
|
|
9
|
+
```
|
|
10
|
+
|
|
11
|
+
Peer dependency: `@framers/agentos` with range `>=0.10.40`.
|
|
12
|
+
|
|
13
|
+
## Usage
|
|
14
|
+
|
|
15
|
+
```ts
|
|
16
|
+
import { AgentOS } from '@framers/agentos';
|
|
17
|
+
|
|
18
|
+
const agentos = await AgentOS.create({
|
|
19
|
+
extensionManifest: {
|
|
20
|
+
packs: [{ package: '@framers/agentos-ext-content-extraction' }],
|
|
21
|
+
},
|
|
22
|
+
});
|
|
23
|
+
```
|
|
24
|
+
|
|
25
|
+
## Tools
|
|
26
|
+
|
|
27
|
+
| Tool | What it does |
|
|
28
|
+
| --- | --- |
|
|
29
|
+
| `extractUrl` | Extracts clean text, markdown or HTML from a URL. |
|
|
30
|
+
| `extractYoutube` | Retrieves a YouTube transcript and video metadata from a video ID or URL. |
|
|
31
|
+
| `extractWikipedia` | Extracts an article's content, summary and metadata in the requested language. |
|
|
32
|
+
| `extractPdf` | Extracts text from a PDF document at a URL. |
|
|
33
|
+
| `extractStructured` | Extracts tables, lists, links and metadata from a web page. |
|
|
34
|
+
|
|
35
|
+
### `extractUrl`
|
|
36
|
+
|
|
37
|
+
| Parameter | Type | Required | Description |
|
|
38
|
+
| --- | --- | --- | --- |
|
|
39
|
+
| `url` | string | yes | URL to extract content from |
|
|
40
|
+
| `format` | string, enum: `text`, `markdown`, `html` | no | Output format (default: `text`) |
|
|
41
|
+
|
|
42
|
+
### `extractYoutube`
|
|
43
|
+
|
|
44
|
+
| Parameter | Type | Required | Description |
|
|
45
|
+
| --- | --- | --- | --- |
|
|
46
|
+
| `videoId` | string | yes | YouTube video ID or full URL (e.g., "dQw4w9WgXcQ" or "https://www.youtube.com/watch?v=dQw4w9WgXcQ") |
|
|
47
|
+
|
|
48
|
+
### `extractWikipedia`
|
|
49
|
+
|
|
50
|
+
| Parameter | Type | Required | Description |
|
|
51
|
+
| --- | --- | --- | --- |
|
|
52
|
+
| `title` | string | yes | Wikipedia article title (e.g., "Artificial intelligence") |
|
|
53
|
+
| `language` | string | no | Wikipedia language code (default: `en`) |
|
|
54
|
+
|
|
55
|
+
### `extractPdf`
|
|
56
|
+
|
|
57
|
+
| Parameter | Type | Required | Description |
|
|
58
|
+
| --- | --- | --- | --- |
|
|
59
|
+
| `url` | string | yes | URL of the PDF document to extract text from |
|
|
60
|
+
|
|
61
|
+
### `extractStructured`
|
|
62
|
+
|
|
63
|
+
| Parameter | Type | Required | Description |
|
|
64
|
+
| --- | --- | --- | --- |
|
|
65
|
+
| `url` | string | yes | URL to extract structured data from |
|
|
66
|
+
| `selectors` | object | no | Optional CSS-style selectors to target specific elements |
|
|
67
|
+
| `selectors.tables` | string | no | Selector for tables |
|
|
68
|
+
| `selectors.lists` | string | no | Selector for lists |
|
|
69
|
+
| `selectors.links` | string | no | Selector for links |
|
|
70
|
+
|
|
71
|
+
## Notes
|
|
72
|
+
|
|
73
|
+
The factory reads no configuration or secrets. Activation initializes the service; calls before initialization return an error. Tools return `success: false` and an `error` message when extraction fails.
|
|
74
|
+
|
|
75
|
+
URL and structured extraction requests time out after `15000` ms. Other HTTP requests use a `30000` ms timeout. Structured extraction returns at most `100` links and ignores the supplied `selectors`.
|
|
76
|
+
|
|
77
|
+
PDF extraction reads text operators from uncompressed streams. When it finds no text, it returns a message explaining that uncompressed text streams are required. YouTube extraction returns `(Transcript not available)` when captions cannot be retrieved.
|
|
78
|
+
|
|
79
|
+
## License
|
|
80
|
+
|
|
81
|
+
Apache-2.0
|
package/manifest.json
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
"$schema": "https://agentos.sh/schemas/extension-manifest-v1.json",
|
|
3
3
|
"id": "com.framers.research.content-extraction",
|
|
4
4
|
"name": "Content Extraction Extension",
|
|
5
|
-
"version": "0.1.
|
|
5
|
+
"version": "0.1.3",
|
|
6
6
|
"description": "Extract clean text, markdown, and structured data from URLs, YouTube videos, Wikipedia articles, and PDF documents.",
|
|
7
7
|
"categories": ["research", "data"],
|
|
8
8
|
"sdk": "axios",
|
package/package.json
CHANGED
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@framers/agentos-ext-content-extraction",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.3",
|
|
4
4
|
"description": "Content extraction extension for AgentOS — extract clean text from URLs, YouTube transcripts, Wikipedia articles, PDFs, and structured data.",
|
|
5
|
+
"license": "Apache-2.0",
|
|
5
6
|
"main": "dist/index.js",
|
|
6
7
|
"types": "dist/index.d.ts",
|
|
7
8
|
"files": [
|
|
@@ -18,7 +19,7 @@
|
|
|
18
19
|
"@types/node": "^20.0.0",
|
|
19
20
|
"typescript": "^5.0.0",
|
|
20
21
|
"vitest": "^3.2.7",
|
|
21
|
-
"@framers/agentos": "^0.
|
|
22
|
+
"@framers/agentos": "^0.13.47"
|
|
22
23
|
},
|
|
23
24
|
"repository": {
|
|
24
25
|
"type": "git",
|