@purepageio/fetch-engines 0.7.1 → 0.7.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md
CHANGED
|
@@ -76,7 +76,7 @@ const result = await fetchStructuredContent("https://example.com/article", schem
|
|
|
76
76
|
console.log(result.data.summary);
|
|
77
77
|
```
|
|
78
78
|
|
|
79
|
-
Set `OPENAI_API_KEY` before running structured helpers, or use `apiConfig` to connect to OpenAI-compatible APIs like OpenRouter:
|
|
79
|
+
Set `OPENAI_API_KEY` (or `OPENROUTER_API_KEY`) before running structured helpers, or use `apiConfig` to connect to OpenAI-compatible APIs like OpenRouter. The engine automatically adds the `Authorization` header when you provide an API key:
|
|
80
80
|
|
|
81
81
|
```typescript
|
|
82
82
|
const result = await fetchStructuredContent("https://example.com/article", schema, {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"StructuredContentEngine.d.ts","sourceRoot":"","sources":["../src/StructuredContentEngine.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAE7B,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,YAAY,CAAC;AAEzD;;GAEG;AACH,MAAM,WAAW,SAAS;IACxB,gFAAgF;IAChF,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,gFAAgF;IAChF,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,gDAAgD;IAChD,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAClC;AAED;;GAEG;AACH,MAAM,WAAW,wBAAwB;IACvC,sJAAsJ;IACtJ,KAAK,EAAE,MAAM,CAAC;IACd,6DAA6D;IAC7D,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,sDAAsD;IACtD,YAAY,CAAC,EAAE,sBAAsB,CAAC;IACtC,2EAA2E;IAC3E,SAAS,CAAC,EAAE,SAAS,CAAC;CACvB;AAED;;GAEG;AACH,MAAM,WAAW,uBAAuB,CAAC,CAAC;IACxC,qDAAqD;IACrD,IAAI,EAAE,CAAC,CAAC;IACR,uDAAuD;IACvD,QAAQ,EAAE,MAAM,CAAC;IACjB,iCAAiC;IACjC,GAAG,EAAE,MAAM,CAAC;IACZ,yCAAyC;IACzC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,8BAA8B;IAC9B,KAAK,EAAE;QACL,YAAY,EAAE,MAAM,CAAC;QACrB,gBAAgB,EAAE,MAAM,CAAC;QACzB,WAAW,EAAE,MAAM,CAAC;KACrB,CAAC;CACH;AAED;;GAEG;AACH,qBAAa,uBAAuB;IAClC,OAAO,CAAC,YAAY,CAAe;gBAEvB,MAAM,GAAE,sBAA2B;IAQ/C;;;;;;;;OAQG;IACG,sBAAsB,CAAC,CAAC,EAC5B,GAAG,EAAE,MAAM,EACX,MAAM,EAAE,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,EACtB,OAAO,EAAE,wBAAwB,GAChC,OAAO,CAAC,uBAAuB,CAAC,CAAC,CAAC,CAAC;
|
|
1
|
+
{"version":3,"file":"StructuredContentEngine.d.ts","sourceRoot":"","sources":["../src/StructuredContentEngine.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAE7B,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,YAAY,CAAC;AAEzD;;GAEG;AACH,MAAM,WAAW,SAAS;IACxB,gFAAgF;IAChF,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,gFAAgF;IAChF,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,gDAAgD;IAChD,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAClC;AAED;;GAEG;AACH,MAAM,WAAW,wBAAwB;IACvC,sJAAsJ;IACtJ,KAAK,EAAE,MAAM,CAAC;IACd,6DAA6D;IAC7D,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,sDAAsD;IACtD,YAAY,CAAC,EAAE,sBAAsB,CAAC;IACtC,2EAA2E;IAC3E,SAAS,CAAC,EAAE,SAAS,CAAC;CACvB;AAED;;GAEG;AACH,MAAM,WAAW,uBAAuB,CAAC,CAAC;IACxC,qDAAqD;IACrD,IAAI,EAAE,CAAC,CAAC;IACR,uDAAuD;IACvD,QAAQ,EAAE,MAAM,CAAC;IACjB,iCAAiC;IACjC,GAAG,EAAE,MAAM,CAAC;IACZ,yCAAyC;IACzC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,8BAA8B;IAC9B,KAAK,EAAE;QACL,YAAY,EAAE,MAAM,CAAC;QACrB,gBAAgB,EAAE,MAAM,CAAC;QACzB,WAAW,EAAE,MAAM,CAAC;KACrB,CAAC;CACH;AAED;;GAEG;AACH,qBAAa,uBAAuB;IAClC,OAAO,CAAC,YAAY,CAAe;gBAEvB,MAAM,GAAE,sBAA2B;IAQ/C;;;;;;;;OAQG;IACG,sBAAsB,CAAC,CAAC,EAC5B,GAAG,EAAE,MAAM,EACX,MAAM,EAAE,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,EACtB,OAAO,EAAE,wBAAwB,GAChC,OAAO,CAAC,uBAAuB,CAAC,CAAC,CAAC,CAAC;IA+DtC;;OAEG;IACH,OAAO,CAAC,cAAc;IAiBtB;;OAEG;IACG,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;CAG/B;AAED;;;;;;;GAOG;AACH,wBAAsB,sBAAsB,CAAC,CAAC,EAC5C,GAAG,EAAE,MAAM,EACX,MAAM,EAAE,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,EACtB,OAAO,EAAE,wBAAwB,GAChC,OAAO,CAAC,uBAAuB,CAAC,CAAC,CAAC,CAAC,CAOrC"}
|
|
@@ -24,9 +24,9 @@ export class StructuredContentEngine {
|
|
|
24
24
|
*/
|
|
25
25
|
async fetchStructuredContent(url, schema, options) {
|
|
26
26
|
const { model, customPrompt = "", engineConfig = {}, apiConfig = {} } = options;
|
|
27
|
-
const apiKey = apiConfig.apiKey ?? process.env.OPENAI_API_KEY;
|
|
27
|
+
const apiKey = apiConfig.apiKey ?? process.env.OPENAI_API_KEY ?? process.env.OPENROUTER_API_KEY;
|
|
28
28
|
if (!apiKey) {
|
|
29
|
-
throw new Error("API key is required for structured content extraction. Provide it via apiConfig.apiKey or set OPENAI_API_KEY environment
|
|
29
|
+
throw new Error("API key is required for structured content extraction. Provide it via apiConfig.apiKey or set OPENAI_API_KEY or OPENROUTER_API_KEY environment variables");
|
|
30
30
|
}
|
|
31
31
|
const result = await this.hybridEngine.fetchHTML(url, {
|
|
32
32
|
markdown: true,
|
|
@@ -42,10 +42,14 @@ ${customPrompt ? `\nAdditional context: ${customPrompt}` : ""}
|
|
|
42
42
|
Content to analyze:
|
|
43
43
|
${result.content}`;
|
|
44
44
|
const modelConfig = this.getModelConfig(model);
|
|
45
|
+
const headers = {
|
|
46
|
+
...(apiConfig.headers ?? {}),
|
|
47
|
+
...(apiConfig.headers?.Authorization ? {} : { Authorization: `Bearer ${apiKey}` }),
|
|
48
|
+
};
|
|
45
49
|
const openai = createOpenAI({
|
|
46
50
|
apiKey,
|
|
47
51
|
...(apiConfig.baseURL && { baseURL: apiConfig.baseURL }),
|
|
48
|
-
|
|
52
|
+
headers,
|
|
49
53
|
});
|
|
50
54
|
try {
|
|
51
55
|
const aiResult = await generateObject({
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"StructuredContentEngine.js","sourceRoot":"","sources":["../src/StructuredContentEngine.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,MAAM,IAAI,CAAC;AACpC,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAE9C,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAiDjD;;GAEG;AACH,MAAM,OAAO,uBAAuB;IAC1B,YAAY,CAAe;IAEnC,YAAY,SAAiC,EAAE;QAC7C,2DAA2D;QAC3D,IAAI,CAAC,YAAY,GAAG,IAAI,YAAY,CAAC;YACnC,GAAG,MAAM;YACT,QAAQ,EAAE,IAAI;SACf,CAAC,CAAC;IACL,CAAC;IAED;;;;;;;;OAQG;IACH,KAAK,CAAC,sBAAsB,CAC1B,GAAW,EACX,MAAsB,EACtB,OAAiC;QAEjC,MAAM,EAAE,KAAK,EAAE,YAAY,GAAG,EAAE,EAAE,YAAY,GAAG,EAAE,EAAE,SAAS,GAAG,EAAE,EAAE,GAAG,OAAO,CAAC;QAEhF,MAAM,MAAM,GAAG,SAAS,CAAC,MAAM,IAAI,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC;
|
|
1
|
+
{"version":3,"file":"StructuredContentEngine.js","sourceRoot":"","sources":["../src/StructuredContentEngine.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,MAAM,IAAI,CAAC;AACpC,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAE9C,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAiDjD;;GAEG;AACH,MAAM,OAAO,uBAAuB;IAC1B,YAAY,CAAe;IAEnC,YAAY,SAAiC,EAAE;QAC7C,2DAA2D;QAC3D,IAAI,CAAC,YAAY,GAAG,IAAI,YAAY,CAAC;YACnC,GAAG,MAAM;YACT,QAAQ,EAAE,IAAI;SACf,CAAC,CAAC;IACL,CAAC;IAED;;;;;;;;OAQG;IACH,KAAK,CAAC,sBAAsB,CAC1B,GAAW,EACX,MAAsB,EACtB,OAAiC;QAEjC,MAAM,EAAE,KAAK,EAAE,YAAY,GAAG,EAAE,EAAE,YAAY,GAAG,EAAE,EAAE,SAAS,GAAG,EAAE,EAAE,GAAG,OAAO,CAAC;QAEhF,MAAM,MAAM,GAAG,SAAS,CAAC,MAAM,IAAI,OAAO,CAAC,GAAG,CAAC,cAAc,IAAI,OAAO,CAAC,GAAG,CAAC,kBAAkB,CAAC;QAChG,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,MAAM,IAAI,KAAK,CACb,0JAA0J,CAC3J,CAAC;QACJ,CAAC;QAED,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,SAAS,CAAC,GAAG,EAAE;YACpD,QAAQ,EAAE,IAAI;YACd,GAAG,YAAY;SAChB,CAAC,CAAC;QAEH,IAAI,MAAM,CAAC,WAAW,KAAK,UAAU,EAAE,CAAC;YACtC,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;QAC3D,CAAC;QAED,MAAM,YAAY,GAAG;;EAEvB,YAAY,CAAC,CAAC,CAAC,yBAAyB,YAAY,EAAE,CAAC,CAAC,CAAC,EAAE;;;EAG3D,MAAM,CAAC,OAAO,EAAE,CAAC;QAEf,MAAM,WAAW,GAAG,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC;QAE/C,MAAM,OAAO,GAAG;YACd,GAAG,CAAC,SAAS,CAAC,OAAO,IAAI,EAAE,CAAC;YAC5B,GAAG,CAAC,SAAS,CAAC,OAAO,EAAE,aAAa,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,aAAa,EAAE,UAAU,MAAM,EAAE,EAAE,CAAC;SACnF,CAAC;QAEF,MAAM,MAAM,GAAG,YAAY,CAAC;YAC1B,MAAM;YACN,GAAG,CAAC,SAAS,CAAC,OAAO,IAAI,EAAE,OAAO,EAAE,SAAS,CAAC,OAAO,EAAE,CAAC;YACxD,OAAO;SACR,CAAC,CAAC;QAEH,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,cAAc,CAAC;gBACpC,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC;gBACpB,MAAM;gBACN,MAAM,EAAE,YAAY;gBACpB,GAAG,WAAW;aACf,CAAC,CAAC;YAEH,OAAO;gBACL,IAAI,EAAE,QAAQ,CAAC,MAAM;gBACrB,QAAQ,EAAE,MAAM,CAAC,OAAO;gBACxB,GAAG,EAAE,MAAM,CAAC,GAAG;gBACf,KAAK,EAAE,MAAM,CAAC,KAAK;gBACnB,KAAK,EAAE;oBACL,YAAY,EAAE,QAAQ,CAAC,KAAK,EAAE,WAAW,IAAI,CAAC;oBAC9C,gBAAgB,EAAE,QAAQ,CAAC,KAAK,EAAE,YAAY,IAAI,CAAC;oBACnD,WAAW,EAAE,QAAQ,CAAC,KAAK,EAAE,WAAW,IAAI,CAAC;iBAC9C;aACF,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CAAC,sCAAsC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QAClH,CAAC;IACH,CAAC;IAED;;OAEG;IACK,cAAc,CAAC,KAAa;QAClC,IAAI,KAAK,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;YAC9B,OAAO;gBACL,eAAe,EAAE;oBACf,MAAM,EAAE;wBACN,gBAAgB,EAAE,KAAK;qBACxB;iBACF;aACF,CAAC;QACJ,CAAC;aAAM,IAAI,KAAK,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;YACvC,OAAO;gBACL,WAAW,EAAE,CAAC;aACf,CAAC;QACJ,CAAC;QACD,OAAO,EAAE,CAAC;IACZ,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,OAAO;QACX,MAAM,IAAI,CAAC,YAAY,CAAC,OAAO,EAAE,CAAC;IACpC,CAAC;CACF;AAED;;;;;;;GAOG;AACH,MAAM,CAAC,KAAK,UAAU,sBAAsB,CAC1C,GAAW,EACX,MAAsB,EACtB,OAAiC;IAEjC,MAAM,MAAM,GAAG,IAAI,uBAAuB,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;IACjE,IAAI,CAAC;QACH,OAAO,MAAM,MAAM,CAAC,sBAAsB,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC;IACnE,CAAC;YAAS,CAAC;QACT,MAAM,MAAM,CAAC,OAAO,EAAE,CAAC;IACzB,CAAC;AACH,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@purepageio/fetch-engines",
|
|
3
|
-
"version": "0.7.
|
|
3
|
+
"version": "0.7.2",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"description": "A collection of configurable engines for fetching HTML content using fetch or Playwright.",
|
|
6
6
|
"main": "dist/index.js",
|
|
@@ -27,7 +27,7 @@
|
|
|
27
27
|
"zod": "^4.1.8"
|
|
28
28
|
},
|
|
29
29
|
"devDependencies": {
|
|
30
|
-
"@playwright/test": "^1.
|
|
30
|
+
"@playwright/test": "^1.57.0",
|
|
31
31
|
"@types/axios": "^0.14.0",
|
|
32
32
|
"@types/jsdom": "^21.1.6",
|
|
33
33
|
"@types/node": "^18.0.0",
|