@purepageio/fetch-engines 0.7.2 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md
CHANGED
|
@@ -65,9 +65,10 @@ await engine.cleanup();
|
|
|
65
65
|
import { fetchStructuredContent } from "@purepageio/fetch-engines";
|
|
66
66
|
import { z } from "zod";
|
|
67
67
|
|
|
68
|
+
// IMPORTANT: All schema fields must have .describe() calls to guide the AI model
|
|
68
69
|
const schema = z.object({
|
|
69
|
-
title: z.string(),
|
|
70
|
-
summary: z.string(),
|
|
70
|
+
title: z.string().describe("The title of the article"),
|
|
71
|
+
summary: z.string().describe("A brief summary of the article content"),
|
|
71
72
|
});
|
|
72
73
|
|
|
73
74
|
// model is required - use any model supported by your API provider
|
|
@@ -92,6 +93,8 @@ const result = await fetchStructuredContent("https://example.com/article", schem
|
|
|
92
93
|
});
|
|
93
94
|
```
|
|
94
95
|
|
|
96
|
+
When you supply a custom `baseURL`, the engine automatically switches to the Vercel AI SDK's `createOpenAICompatible` provider (instead of `createOpenAI`) so OpenAI-compatible services like OpenRouter receive the correct API-key auth flow.
|
|
97
|
+
|
|
95
98
|
## Configuration
|
|
96
99
|
|
|
97
100
|
### Essentials
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import
|
|
1
|
+
import { z } from "zod";
|
|
2
2
|
import type { PlaywrightEngineConfig } from "./types.js";
|
|
3
3
|
/**
|
|
4
4
|
* Configuration for OpenAI-compatible API providers
|
|
@@ -49,16 +49,27 @@ export interface StructuredContentResult<T> {
|
|
|
49
49
|
export declare class StructuredContentEngine {
|
|
50
50
|
private hybridEngine;
|
|
51
51
|
constructor(config?: PlaywrightEngineConfig);
|
|
52
|
+
/**
|
|
53
|
+
* Extracts field descriptions from a Zod object schema
|
|
54
|
+
* @private
|
|
55
|
+
*/
|
|
56
|
+
private extractFieldDescriptions;
|
|
57
|
+
/**
|
|
58
|
+
* Validates that all fields in the schema have descriptions
|
|
59
|
+
* @private
|
|
60
|
+
*/
|
|
61
|
+
private validateSchemaDescriptions;
|
|
52
62
|
/**
|
|
53
63
|
* Fetches content from a URL and extracts structured data using AI
|
|
54
64
|
*
|
|
55
65
|
* @param url The URL to fetch content from
|
|
56
|
-
* @param schema Zod schema defining the structure of data to extract
|
|
66
|
+
* @param schema Zod schema defining the structure of data to extract (all fields must have .describe() calls)
|
|
57
67
|
* @param options Additional options for the extraction process
|
|
58
68
|
* @returns Promise resolving to structured data and metadata
|
|
59
|
-
* @throws Error if API key is not set or if extraction fails
|
|
69
|
+
* @throws Error if API key is not set, if schema fields lack descriptions, or if extraction fails
|
|
60
70
|
*/
|
|
61
71
|
fetchStructuredContent<T>(url: string, schema: z.ZodSchema<T>, options: StructuredContentOptions): Promise<StructuredContentResult<T>>;
|
|
72
|
+
private getOpenAIProvider;
|
|
62
73
|
/**
|
|
63
74
|
* Get model-specific configuration options
|
|
64
75
|
*/
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"StructuredContentEngine.d.ts","sourceRoot":"","sources":["../src/StructuredContentEngine.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"StructuredContentEngine.d.ts","sourceRoot":"","sources":["../src/StructuredContentEngine.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,YAAY,CAAC;AAQzD;;GAEG;AACH,MAAM,WAAW,SAAS;IACxB,gFAAgF;IAChF,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,gFAAgF;IAChF,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,gDAAgD;IAChD,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAClC;AAED;;GAEG;AACH,MAAM,WAAW,wBAAwB;IACvC,sJAAsJ;IACtJ,KAAK,EAAE,MAAM,CAAC;IACd,6DAA6D;IAC7D,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,sDAAsD;IACtD,YAAY,CAAC,EAAE,sBAAsB,CAAC;IACtC,2EAA2E;IAC3E,SAAS,CAAC,EAAE,SAAS,CAAC;CACvB;AAED;;GAEG;AACH,MAAM,WAAW,uBAAuB,CAAC,CAAC;IACxC,qDAAqD;IACrD,IAAI,EAAE,CAAC,CAAC;IACR,uDAAuD;IACvD,QAAQ,EAAE,MAAM,CAAC;IACjB,iCAAiC;IACjC,GAAG,EAAE,MAAM,CAAC;IACZ,yCAAyC;IACzC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,8BAA8B;IAC9B,KAAK,EAAE;QACL,YAAY,EAAE,MAAM,CAAC;QACrB,gBAAgB,EAAE,MAAM,CAAC;QACzB,WAAW,EAAE,MAAM,CAAC;KACrB,CAAC;CACH;AAED;;GAEG;AACH,qBAAa,uBAAuB;IAClC,OAAO,CAAC,YAAY,CAAe;gBAEvB,MAAM,GAAE,sBAA2B;IAQ/C;;;OAGG;IACH,OAAO,CAAC,wBAAwB;IA6BhC;;;OAGG;IACH,OAAO,CAAC,0BAA0B;IA2ClC;;;;;;;;OAQG;IACG,sBAAsB,CAAC,CAAC,EAC5B,GAAG,EAAE,MAAM,EACX,MAAM,EAAE,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,EACtB,OAAO,EAAE,wBAAwB,GAChC,OAAO,CAAC,uBAAuB,CAAC,CAAC,CAAC,CAAC;IAyLtC,OAAO,CAAC,iBAAiB;IAoBzB;;OAEG;IACH,OAAO,CAAC,cAAc;IAiBtB;;OAEG;IACG,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;CAG/B;AAED;;;;;;;GAOG;AACH,wBAAsB,sBAAsB,CAAC,CAAC,EAC5C,GAAG,EAAE,MAAM,EACX,MAAM,EAAE,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,EACtB,OAAO,EAAE,wBAAwB,GAChC,OAAO,CAAC,uBAAuB,CAAC,CAAC,CAAC,CAAC,CAOrC"}
|
|
@@ -1,6 +1,13 @@
|
|
|
1
|
-
import { generateObject } from "ai";
|
|
1
|
+
import { generateObject, NoObjectGeneratedError } from "ai";
|
|
2
2
|
import { createOpenAI } from "@ai-sdk/openai";
|
|
3
|
+
import { createOpenAICompatible } from "@ai-sdk/openai-compatible";
|
|
4
|
+
import { z } from "zod";
|
|
3
5
|
import { HybridEngine } from "./HybridEngine.js";
|
|
6
|
+
// Suppress AI SDK warnings about responseFormat/structuredOutputs
|
|
7
|
+
// This warning appears when using OpenAI-compatible APIs that don't support structured outputs
|
|
8
|
+
if (typeof globalThis !== "undefined") {
|
|
9
|
+
globalThis.AI_SDK_LOG_WARNINGS = false;
|
|
10
|
+
}
|
|
4
11
|
/**
|
|
5
12
|
* Engine for fetching web content and extracting structured data using AI
|
|
6
13
|
*/
|
|
@@ -13,14 +20,82 @@ export class StructuredContentEngine {
|
|
|
13
20
|
markdown: true,
|
|
14
21
|
});
|
|
15
22
|
}
|
|
23
|
+
/**
|
|
24
|
+
* Extracts field descriptions from a Zod object schema
|
|
25
|
+
* @private
|
|
26
|
+
*/
|
|
27
|
+
extractFieldDescriptions(schema) {
|
|
28
|
+
const descriptions = new Map();
|
|
29
|
+
if (schema instanceof z.ZodObject) {
|
|
30
|
+
const shape = schema.shape;
|
|
31
|
+
for (const [key, fieldSchema] of Object.entries(shape)) {
|
|
32
|
+
// In Zod v4, description is stored directly on the schema object
|
|
33
|
+
let description;
|
|
34
|
+
const fieldSchemaAny = fieldSchema;
|
|
35
|
+
const def = fieldSchemaAny._def;
|
|
36
|
+
// Handle optional fields - check innerType for description
|
|
37
|
+
// When using .describe().optional(), description is on innerType
|
|
38
|
+
// When using .optional().describe(), description is on the optional wrapper
|
|
39
|
+
if (def?.typeName === "ZodOptional") {
|
|
40
|
+
description = fieldSchemaAny._def?.innerType?.description || fieldSchemaAny.description;
|
|
41
|
+
}
|
|
42
|
+
else {
|
|
43
|
+
description = fieldSchemaAny.description;
|
|
44
|
+
}
|
|
45
|
+
if (description) {
|
|
46
|
+
descriptions.set(key, description);
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
return descriptions;
|
|
51
|
+
}
|
|
52
|
+
/**
|
|
53
|
+
* Validates that all fields in the schema have descriptions
|
|
54
|
+
* @private
|
|
55
|
+
*/
|
|
56
|
+
validateSchemaDescriptions(schema) {
|
|
57
|
+
if (!(schema instanceof z.ZodObject)) {
|
|
58
|
+
throw new Error("Schema must be a Zod object. Use z.object() to define your schema structure.");
|
|
59
|
+
}
|
|
60
|
+
const shape = schema.shape;
|
|
61
|
+
const missingDescriptions = [];
|
|
62
|
+
for (const [key, fieldSchema] of Object.entries(shape)) {
|
|
63
|
+
const fieldSchemaAny = fieldSchema;
|
|
64
|
+
// In Zod v4, description is stored directly on the schema object
|
|
65
|
+
// Check if it's an optional field by checking for innerType
|
|
66
|
+
let hasDescription = false;
|
|
67
|
+
// Check if this is an optional field (has innerType)
|
|
68
|
+
if (fieldSchemaAny._def?.innerType) {
|
|
69
|
+
// For optional fields:
|
|
70
|
+
// - When using .describe().optional(), description is on innerType
|
|
71
|
+
// - When using .optional().describe(), description is on the optional wrapper
|
|
72
|
+
hasDescription = !!(fieldSchemaAny._def.innerType.description || fieldSchemaAny.description);
|
|
73
|
+
}
|
|
74
|
+
else {
|
|
75
|
+
// For non-optional fields, description is directly on the schema
|
|
76
|
+
hasDescription = !!fieldSchemaAny.description;
|
|
77
|
+
}
|
|
78
|
+
if (!hasDescription) {
|
|
79
|
+
missingDescriptions.push(key);
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
if (missingDescriptions.length > 0) {
|
|
83
|
+
throw new Error(`All schema fields must have descriptions. Missing descriptions for: ${missingDescriptions.join(", ")}\n\n` +
|
|
84
|
+
`Example:\n` +
|
|
85
|
+
`z.object({\n` +
|
|
86
|
+
` ${missingDescriptions[0]}: z.string().describe("Description of ${missingDescriptions[0]}"),\n` +
|
|
87
|
+
` // ... other fields\n` +
|
|
88
|
+
`})`);
|
|
89
|
+
}
|
|
90
|
+
}
|
|
16
91
|
/**
|
|
17
92
|
* Fetches content from a URL and extracts structured data using AI
|
|
18
93
|
*
|
|
19
94
|
* @param url The URL to fetch content from
|
|
20
|
-
* @param schema Zod schema defining the structure of data to extract
|
|
95
|
+
* @param schema Zod schema defining the structure of data to extract (all fields must have .describe() calls)
|
|
21
96
|
* @param options Additional options for the extraction process
|
|
22
97
|
* @returns Promise resolving to structured data and metadata
|
|
23
|
-
* @throws Error if API key is not set or if extraction fails
|
|
98
|
+
* @throws Error if API key is not set, if schema fields lack descriptions, or if extraction fails
|
|
24
99
|
*/
|
|
25
100
|
async fetchStructuredContent(url, schema, options) {
|
|
26
101
|
const { model, customPrompt = "", engineConfig = {}, apiConfig = {} } = options;
|
|
@@ -35,8 +110,25 @@ export class StructuredContentEngine {
|
|
|
35
110
|
if (result.contentType !== "markdown") {
|
|
36
111
|
throw new Error("Failed to convert content to markdown");
|
|
37
112
|
}
|
|
113
|
+
// Validate that all fields have descriptions
|
|
114
|
+
this.validateSchemaDescriptions(schema);
|
|
115
|
+
// Extract field descriptions to include in prompt
|
|
116
|
+
const fieldDescriptions = this.extractFieldDescriptions(schema);
|
|
117
|
+
const schemaGuidance = Array.from(fieldDescriptions.entries())
|
|
118
|
+
.map(([field, description]) => `- ${field}: ${description}`)
|
|
119
|
+
.join("\n");
|
|
38
120
|
const systemPrompt = `You are an expert at extracting structured data from web content.
|
|
39
121
|
Extract the requested information from the provided markdown content accurately and completely.
|
|
122
|
+
Return the data as a valid JSON object matching the exact schema provided.
|
|
123
|
+
|
|
124
|
+
Field requirements:
|
|
125
|
+
${schemaGuidance}
|
|
126
|
+
|
|
127
|
+
IMPORTANT: Pay careful attention to data types:
|
|
128
|
+
- Numbers should be returned as numeric values (not strings with currency symbols)
|
|
129
|
+
- Strings should be returned as plain text strings
|
|
130
|
+
- Follow the exact schema structure and field descriptions provided above
|
|
131
|
+
|
|
40
132
|
${customPrompt ? `\nAdditional context: ${customPrompt}` : ""}
|
|
41
133
|
|
|
42
134
|
Content to analyze:
|
|
@@ -46,9 +138,9 @@ ${result.content}`;
|
|
|
46
138
|
...(apiConfig.headers ?? {}),
|
|
47
139
|
...(apiConfig.headers?.Authorization ? {} : { Authorization: `Bearer ${apiKey}` }),
|
|
48
140
|
};
|
|
49
|
-
const openai =
|
|
141
|
+
const openai = this.getOpenAIProvider({
|
|
50
142
|
apiKey,
|
|
51
|
-
|
|
143
|
+
baseURL: apiConfig.baseURL,
|
|
52
144
|
headers,
|
|
53
145
|
});
|
|
54
146
|
try {
|
|
@@ -71,9 +163,109 @@ ${result.content}`;
|
|
|
71
163
|
};
|
|
72
164
|
}
|
|
73
165
|
catch (error) {
|
|
166
|
+
if (NoObjectGeneratedError.isInstance(error)) {
|
|
167
|
+
const errorMessage = `Failed to extract structured data: ${error.message}`;
|
|
168
|
+
const details = [];
|
|
169
|
+
// Get expected keys from schema
|
|
170
|
+
const expectedKeys = schema instanceof z.ZodObject ? Object.keys(schema.shape) : [];
|
|
171
|
+
if (error.text) {
|
|
172
|
+
// Try to parse and format the JSON response if possible
|
|
173
|
+
let formattedResponse = error.text;
|
|
174
|
+
let hasWeirdKeys = false;
|
|
175
|
+
let receivedKeys = [];
|
|
176
|
+
try {
|
|
177
|
+
const parsed = JSON.parse(error.text);
|
|
178
|
+
formattedResponse = JSON.stringify(parsed, null, 2);
|
|
179
|
+
receivedKeys = Object.keys(parsed);
|
|
180
|
+
// Check if keys have weird prefixes (like dots) or don't match expected keys
|
|
181
|
+
hasWeirdKeys = receivedKeys.some((key) => key.startsWith(".") || key.includes(" ") || (expectedKeys.length > 0 && !expectedKeys.includes(key)));
|
|
182
|
+
}
|
|
183
|
+
catch {
|
|
184
|
+
// Not valid JSON, show raw text
|
|
185
|
+
formattedResponse = error.text.substring(0, 300) + (error.text.length > 300 ? "..." : "");
|
|
186
|
+
}
|
|
187
|
+
if (hasWeirdKeys && expectedKeys.length > 0) {
|
|
188
|
+
details.push(`⚠️ Key mismatch detected:`);
|
|
189
|
+
details.push(` Expected: ${expectedKeys.join(", ")}`);
|
|
190
|
+
details.push(` Received: ${receivedKeys.join(", ")}`);
|
|
191
|
+
details.push(`\n The model returned keys with unexpected format (possibly prefixed with dots or spaces).`);
|
|
192
|
+
details.push(` This usually indicates the model doesn't properly support structured outputs.`);
|
|
193
|
+
}
|
|
194
|
+
details.push(`Model response:\n${formattedResponse}`);
|
|
195
|
+
}
|
|
196
|
+
if (error.finishReason && error.finishReason !== "unknown") {
|
|
197
|
+
details.push(`Finish reason: ${error.finishReason}`);
|
|
198
|
+
}
|
|
199
|
+
if (error.cause) {
|
|
200
|
+
const causeMessage = error.cause instanceof Error ? error.cause.message : String(error.cause);
|
|
201
|
+
// Parse Zod validation errors if present
|
|
202
|
+
if (causeMessage.includes("Type validation failed")) {
|
|
203
|
+
try {
|
|
204
|
+
// Try to extract the error message array from the cause
|
|
205
|
+
const zodErrorsMatch = causeMessage.match(/Error message:\s*(\[[\s\S]*\])/);
|
|
206
|
+
if (zodErrorsMatch) {
|
|
207
|
+
const zodErrors = JSON.parse(zodErrorsMatch[1]);
|
|
208
|
+
const formattedErrors = zodErrors
|
|
209
|
+
.map((err) => {
|
|
210
|
+
const path = err.path.length > 0 ? err.path.join(".") : "root";
|
|
211
|
+
const received = err.received || "unknown";
|
|
212
|
+
return ` • ${path}: ${err.message} (expected ${err.expected}, received ${received})`;
|
|
213
|
+
})
|
|
214
|
+
.join("\n");
|
|
215
|
+
details.push(`\nSchema validation errors:\n${formattedErrors}`);
|
|
216
|
+
// Detect common type mismatches and provide helpful suggestions
|
|
217
|
+
const typeMismatches = zodErrors.filter((err) => err.expected === "number" && err.received === "string");
|
|
218
|
+
if (typeMismatches.length > 0) {
|
|
219
|
+
const mismatchedFields = typeMismatches
|
|
220
|
+
.map((err) => err.path.join("."))
|
|
221
|
+
.join(", ");
|
|
222
|
+
details.push(`\n💡 Tip: Fields (${mismatchedFields}) are being returned as strings but expected as numbers.`);
|
|
223
|
+
details.push(` Consider using Zod transforms to handle currency strings or other formatted numbers:`);
|
|
224
|
+
details.push(` Example: z.string().transform((val) => parseFloat(val.replace(/[^0-9.-]/g, "")))`);
|
|
225
|
+
}
|
|
226
|
+
}
|
|
227
|
+
else {
|
|
228
|
+
// Fallback: show the cause message
|
|
229
|
+
const shortCause = causeMessage.length > 200 ? causeMessage.substring(0, 200) + "..." : causeMessage;
|
|
230
|
+
details.push(`\nValidation error: ${shortCause}`);
|
|
231
|
+
}
|
|
232
|
+
}
|
|
233
|
+
catch {
|
|
234
|
+
// If parsing fails, show a shortened version
|
|
235
|
+
const shortCause = causeMessage.length > 300 ? causeMessage.substring(0, 300) + "..." : causeMessage;
|
|
236
|
+
details.push(`\nCause: ${shortCause}`);
|
|
237
|
+
}
|
|
238
|
+
}
|
|
239
|
+
else {
|
|
240
|
+
const shortCause = causeMessage.length > 300 ? causeMessage.substring(0, 300) + "..." : causeMessage;
|
|
241
|
+
details.push(`\nCause: ${shortCause}`);
|
|
242
|
+
}
|
|
243
|
+
}
|
|
244
|
+
const fullMessage = details.length > 0
|
|
245
|
+
? `${errorMessage}\n\n${details.join("\n")}\n\n💡 Tip: This model may not support structured outputs properly. Try using a model that supports structured outputs (e.g., gpt-4o-mini, gpt-4o, claude-3.5-sonnet).`
|
|
246
|
+
: errorMessage;
|
|
247
|
+
throw new Error(fullMessage);
|
|
248
|
+
}
|
|
74
249
|
throw new Error(`Failed to extract structured data: ${error instanceof Error ? error.message : String(error)}`);
|
|
75
250
|
}
|
|
76
251
|
}
|
|
252
|
+
getOpenAIProvider(params) {
|
|
253
|
+
const normalizedBaseURL = params.baseURL?.replace(/\/+$/, "");
|
|
254
|
+
const defaultOpenAIBaseURL = "https://api.openai.com/v1";
|
|
255
|
+
if (normalizedBaseURL && normalizedBaseURL !== defaultOpenAIBaseURL) {
|
|
256
|
+
return createOpenAICompatible({
|
|
257
|
+
name: "openai-compatible",
|
|
258
|
+
apiKey: params.apiKey,
|
|
259
|
+
baseURL: normalizedBaseURL,
|
|
260
|
+
headers: params.headers,
|
|
261
|
+
});
|
|
262
|
+
}
|
|
263
|
+
return createOpenAI({
|
|
264
|
+
apiKey: params.apiKey,
|
|
265
|
+
...(normalizedBaseURL && { baseURL: normalizedBaseURL }),
|
|
266
|
+
headers: params.headers,
|
|
267
|
+
});
|
|
268
|
+
}
|
|
77
269
|
/**
|
|
78
270
|
* Get model-specific configuration options
|
|
79
271
|
*/
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"StructuredContentEngine.js","sourceRoot":"","sources":["../src/StructuredContentEngine.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,MAAM,IAAI,CAAC;
|
|
1
|
+
{"version":3,"file":"StructuredContentEngine.js","sourceRoot":"","sources":["../src/StructuredContentEngine.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,sBAAsB,EAAE,MAAM,IAAI,CAAC;AAC5D,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EAAE,sBAAsB,EAAE,MAAM,2BAA2B,CAAC;AACnE,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAGjD,kEAAkE;AAClE,+FAA+F;AAC/F,IAAI,OAAO,UAAU,KAAK,WAAW,EAAE,CAAC;IACrC,UAAkB,CAAC,mBAAmB,GAAG,KAAK,CAAC;AAClD,CAAC;AAgDD;;GAEG;AACH,MAAM,OAAO,uBAAuB;IAC1B,YAAY,CAAe;IAEnC,YAAY,SAAiC,EAAE;QAC7C,2DAA2D;QAC3D,IAAI,CAAC,YAAY,GAAG,IAAI,YAAY,CAAC;YACnC,GAAG,MAAM;YACT,QAAQ,EAAE,IAAI;SACf,CAAC,CAAC;IACL,CAAC;IAED;;;OAGG;IACK,wBAAwB,CAAC,MAAmB;QAClD,MAAM,YAAY,GAAG,IAAI,GAAG,EAAkB,CAAC;QAE/C,IAAI,MAAM,YAAY,CAAC,CAAC,SAAS,EAAE,CAAC;YAClC,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC;YAC3B,KAAK,MAAM,CAAC,GAAG,EAAE,WAAW,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;gBACvD,iEAAiE;gBACjE,IAAI,WAA+B,CAAC;gBACpC,MAAM,cAAc,GAAG,WAAkB,CAAC;gBAC1C,MAAM,GAAG,GAAG,cAAc,CAAC,IAAI,CAAC;gBAEhC,2DAA2D;gBAC3D,iEAAiE;gBACjE,4EAA4E;gBAC5E,IAAI,GAAG,EAAE,QAAQ,KAAK,aAAa,EAAE,CAAC;oBACpC,WAAW,GAAG,cAAc,CAAC,IAAI,EAAE,SAAS,EAAE,WAAW,IAAI,cAAc,CAAC,WAAW,CAAC;gBAC1F,CAAC;qBAAM,CAAC;oBACN,WAAW,GAAG,cAAc,CAAC,WAAW,CAAC;gBAC3C,CAAC;gBAED,IAAI,WAAW,EAAE,CAAC;oBAChB,YAAY,CAAC,GAAG,CAAC,GAAG,EAAE,WAAW,CAAC,CAAC;gBACrC,CAAC;YACH,CAAC;QACH,CAAC;QAED,OAAO,YAAY,CAAC;IACtB,CAAC;IAED;;;OAGG;IACK,0BAA0B,CAAC,MAAmB;QACpD,IAAI,CAAC,CAAC,MAAM,YAAY,CAAC,CAAC,SAAS,CAAC,EAAE,CAAC;YACrC,MAAM,IAAI,KAAK,CAAC,8EAA8E,CAAC,CAAC;QAClG,CAAC;QAED,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC;QAC3B,MAAM,mBAAmB,GAAa,EAAE,CAAC;QAEzC,KAAK,MAAM,CAAC,GAAG,EAAE,WAAW,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;YACvD,MAAM,cAAc,GAAG,WAAkB,CAAC;YAE1C,iEAAiE;YACjE,4DAA4D;YAC5D,IAAI,cAAc,GAAG,KAAK,CAAC;YAE3B,qDAAqD;YACrD,IAAI,cAAc,CAAC,IAAI,EAAE,SAAS,EAAE,CAAC;gBACnC,uBAAuB;gBACvB,mEAAmE;gBACnE,8EAA8E;gBAC9E,cAAc,GAAG,CAAC,CAAC,CAAC,cAAc,CAAC,IAAI,CAAC,SAAS,CAAC,WAAW,IAAI,cAAc,CAAC,WAAW,CAAC,CAAC;YAC/F,CAAC;iBAAM,CAAC;gBACN,iEAAiE;gBACjE,cAAc,GAAG,CAAC,CAAC,cAAc,CAAC,WAAW,CAAC;YAChD,CAAC;YAED,IAAI,CAAC,cAAc,EAAE,CAAC;gBACpB,mBAAmB,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAChC,CAAC;QACH,CAAC;QAED,IAAI,mBAAmB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACnC,MAAM,IAAI,KAAK,CACb,uEAAuE,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM;gBACzG,YAAY;gBACZ,cAAc;gBACd,KAAK,mBAAmB,CAAC,CAAC,CAAC,yCAAyC,mBAAmB,CAAC,CAAC,CAAC,OAAO;gBACjG,yBAAyB;gBACzB,IAAI,CACP,CAAC;QACJ,CAAC;IACH,CAAC;IAED;;;;;;;;OAQG;IACH,KAAK,CAAC,sBAAsB,CAC1B,GAAW,EACX,MAAsB,EACtB,OAAiC;QAEjC,MAAM,EAAE,KAAK,EAAE,YAAY,GAAG,EAAE,EAAE,YAAY,GAAG,EAAE,EAAE,SAAS,GAAG,EAAE,EAAE,GAAG,OAAO,CAAC;QAEhF,MAAM,MAAM,GAAG,SAAS,CAAC,MAAM,IAAI,OAAO,CAAC,GAAG,CAAC,cAAc,IAAI,OAAO,CAAC,GAAG,CAAC,kBAAkB,CAAC;QAChG,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,MAAM,IAAI,KAAK,CACb,0JAA0J,CAC3J,CAAC;QACJ,CAAC;QAED,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,SAAS,CAAC,GAAG,EAAE;YACpD,QAAQ,EAAE,IAAI;YACd,GAAG,YAAY;SAChB,CAAC,CAAC;QAEH,IAAI,MAAM,CAAC,WAAW,KAAK,UAAU,EAAE,CAAC;YACtC,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;QAC3D,CAAC;QAED,6CAA6C;QAC7C,IAAI,CAAC,0BAA0B,CAAC,MAAM,CAAC,CAAC;QAExC,kDAAkD;QAClD,MAAM,iBAAiB,GAAG,IAAI,CAAC,wBAAwB,CAAC,MAAM,CAAC,CAAC;QAChE,MAAM,cAAc,GAAG,KAAK,CAAC,IAAI,CAAC,iBAAiB,CAAC,OAAO,EAAE,CAAC;aAC3D,GAAG,CAAC,CAAC,CAAC,KAAK,EAAE,WAAW,CAAC,EAAE,EAAE,CAAC,KAAK,KAAK,KAAK,WAAW,EAAE,CAAC;aAC3D,IAAI,CAAC,IAAI,CAAC,CAAC;QAEd,MAAM,YAAY,GAAG;;;;;EAKvB,cAAc;;;;;;;EAOd,YAAY,CAAC,CAAC,CAAC,yBAAyB,YAAY,EAAE,CAAC,CAAC,CAAC,EAAE;;;EAG3D,MAAM,CAAC,OAAO,EAAE,CAAC;QAEf,MAAM,WAAW,GAAG,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC;QAE/C,MAAM,OAAO,GAAG;YACd,GAAG,CAAC,SAAS,CAAC,OAAO,IAAI,EAAE,CAAC;YAC5B,GAAG,CAAC,SAAS,CAAC,OAAO,EAAE,aAAa,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,aAAa,EAAE,UAAU,MAAM,EAAE,EAAE,CAAC;SACnF,CAAC;QAEF,MAAM,MAAM,GAAG,IAAI,CAAC,iBAAiB,CAAC;YACpC,MAAM;YACN,OAAO,EAAE,SAAS,CAAC,OAAO;YAC1B,OAAO;SACR,CAAC,CAAC;QAEH,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,cAAc,CAAC;gBACpC,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC;gBACpB,MAAM;gBACN,MAAM,EAAE,YAAY;gBACpB,GAAG,WAAW;aACf,CAAC,CAAC;YAEH,OAAO;gBACL,IAAI,EAAE,QAAQ,CAAC,MAAM;gBACrB,QAAQ,EAAE,MAAM,CAAC,OAAO;gBACxB,GAAG,EAAE,MAAM,CAAC,GAAG;gBACf,KAAK,EAAE,MAAM,CAAC,KAAK;gBACnB,KAAK,EAAE;oBACL,YAAY,EAAE,QAAQ,CAAC,KAAK,EAAE,WAAW,IAAI,CAAC;oBAC9C,gBAAgB,EAAE,QAAQ,CAAC,KAAK,EAAE,YAAY,IAAI,CAAC;oBACnD,WAAW,EAAE,QAAQ,CAAC,KAAK,EAAE,WAAW,IAAI,CAAC;iBAC9C;aACF,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,sBAAsB,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;gBAC7C,MAAM,YAAY,GAAG,sCAAsC,KAAK,CAAC,OAAO,EAAE,CAAC;gBAC3E,MAAM,OAAO,GAAa,EAAE,CAAC;gBAE7B,gCAAgC;gBAChC,MAAM,YAAY,GAAG,MAAM,YAAY,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;gBAEpF,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC;oBACf,wDAAwD;oBACxD,IAAI,iBAAiB,GAAG,KAAK,CAAC,IAAI,CAAC;oBACnC,IAAI,YAAY,GAAG,KAAK,CAAC;oBACzB,IAAI,YAAY,GAAa,EAAE,CAAC;oBAEhC,IAAI,CAAC;wBACH,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;wBACtC,iBAAiB,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;wBACpD,YAAY,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;wBACnC,6EAA6E;wBAC7E,YAAY,GAAG,YAAY,CAAC,IAAI,CAC9B,CAAC,GAAG,EAAE,EAAE,CACN,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CACvG,CAAC;oBACJ,CAAC;oBAAC,MAAM,CAAC;wBACP,gCAAgC;wBAChC,iBAAiB,GAAG,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;oBAC5F,CAAC;oBAED,IAAI,YAAY,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;wBAC5C,OAAO,CAAC,IAAI,CAAC,4BAA4B,CAAC,CAAC;wBAC3C,OAAO,CAAC,IAAI,CAAC,gBAAgB,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;wBACxD,OAAO,CAAC,IAAI,CAAC,gBAAgB,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;wBACxD,OAAO,CAAC,IAAI,CACV,8FAA8F,CAC/F,CAAC;wBACF,OAAO,CAAC,IAAI,CAAC,kFAAkF,CAAC,CAAC;oBACnG,CAAC;oBAED,OAAO,CAAC,IAAI,CAAC,oBAAoB,iBAAiB,EAAE,CAAC,CAAC;gBACxD,CAAC;gBAED,IAAI,KAAK,CAAC,YAAY,IAAI,KAAK,CAAC,YAAY,KAAK,SAAS,EAAE,CAAC;oBAC3D,OAAO,CAAC,IAAI,CAAC,kBAAkB,KAAK,CAAC,YAAY,EAAE,CAAC,CAAC;gBACvD,CAAC;gBAED,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC;oBAChB,MAAM,YAAY,GAAG,KAAK,CAAC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;oBAC9F,yCAAyC;oBACzC,IAAI,YAAY,CAAC,QAAQ,CAAC,wBAAwB,CAAC,EAAE,CAAC;wBACpD,IAAI,CAAC;4BACH,wDAAwD;4BACxD,MAAM,cAAc,GAAG,YAAY,CAAC,KAAK,CAAC,gCAAgC,CAAC,CAAC;4BAC5E,IAAI,cAAc,EAAE,CAAC;gCACnB,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,CAAC;gCAChD,MAAM,eAAe,GAAG,SAAS;qCAC9B,GAAG,CAAC,CAAC,GAA4E,EAAE,EAAE;oCACpF,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;oCAC/D,MAAM,QAAQ,GAAG,GAAG,CAAC,QAAQ,IAAI,SAAS,CAAC;oCAC3C,OAAO,OAAO,IAAI,KAAK,GAAG,CAAC,OAAO,cAAc,GAAG,CAAC,QAAQ,cAAc,QAAQ,GAAG,CAAC;gCACxF,CAAC,CAAC;qCACD,IAAI,CAAC,IAAI,CAAC,CAAC;gCACd,OAAO,CAAC,IAAI,CAAC,gCAAgC,eAAe,EAAE,CAAC,CAAC;gCAEhE,gEAAgE;gCAChE,MAAM,cAAc,GAAG,SAAS,CAAC,MAAM,CACrC,CAAC,GAA2C,EAAE,EAAE,CAC9C,GAAG,CAAC,QAAQ,KAAK,QAAQ,IAAI,GAAG,CAAC,QAAQ,KAAK,QAAQ,CACzD,CAAC;gCACF,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oCAC9B,MAAM,gBAAgB,GAAG,cAAc;yCACpC,GAAG,CAAC,CAAC,GAAuB,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;yCACpD,IAAI,CAAC,IAAI,CAAC,CAAC;oCACd,OAAO,CAAC,IAAI,CACV,qBAAqB,gBAAgB,0DAA0D,CAChG,CAAC;oCACF,OAAO,CAAC,IAAI,CACV,yFAAyF,CAC1F,CAAC;oCACF,OAAO,CAAC,IAAI,CAAC,qFAAqF,CAAC,CAAC;gCACtG,CAAC;4BACH,CAAC;iCAAM,CAAC;gCACN,mCAAmC;gCACnC,MAAM,UAAU,GAAG,YAAY,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC,EAAE,GAAG,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,YAAY,CAAC;gCACrG,OAAO,CAAC,IAAI,CAAC,uBAAuB,UAAU,EAAE,CAAC,CAAC;4BACpD,CAAC;wBACH,CAAC;wBAAC,MAAM,CAAC;4BACP,6CAA6C;4BAC7C,MAAM,UAAU,GAAG,YAAY,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC,EAAE,GAAG,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,YAAY,CAAC;4BACrG,OAAO,CAAC,IAAI,CAAC,YAAY,UAAU,EAAE,CAAC,CAAC;wBACzC,CAAC;oBACH,CAAC;yBAAM,CAAC;wBACN,MAAM,UAAU,GAAG,YAAY,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC,EAAE,GAAG,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,YAAY,CAAC;wBACrG,OAAO,CAAC,IAAI,CAAC,YAAY,UAAU,EAAE,CAAC,CAAC;oBACzC,CAAC;gBACH,CAAC;gBAED,MAAM,WAAW,GACf,OAAO,CAAC,MAAM,GAAG,CAAC;oBAChB,CAAC,CAAC,GAAG,YAAY,OAAO,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,wKAAwK;oBAClN,CAAC,CAAC,YAAY,CAAC;gBAEnB,MAAM,IAAI,KAAK,CAAC,WAAW,CAAC,CAAC;YAC/B,CAAC;YAED,MAAM,IAAI,KAAK,CAAC,sCAAsC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QAClH,CAAC;IACH,CAAC;IAEO,iBAAiB,CAAC,MAA6E;QACrG,MAAM,iBAAiB,GAAG,MAAM,CAAC,OAAO,EAAE,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;QAC9D,MAAM,oBAAoB,GAAG,2BAA2B,CAAC;QAEzD,IAAI,iBAAiB,IAAI,iBAAiB,KAAK,oBAAoB,EAAE,CAAC;YACpE,OAAO,sBAAsB,CAAC;gBAC5B,IAAI,EAAE,mBAAmB;gBACzB,MAAM,EAAE,MAAM,CAAC,MAAM;gBACrB,OAAO,EAAE,iBAAiB;gBAC1B,OAAO,EAAE,MAAM,CAAC,OAAO;aACxB,CAAC,CAAC;QACL,CAAC;QAED,OAAO,YAAY,CAAC;YAClB,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,GAAG,CAAC,iBAAiB,IAAI,EAAE,OAAO,EAAE,iBAAiB,EAAE,CAAC;YACxD,OAAO,EAAE,MAAM,CAAC,OAAO;SACxB,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACK,cAAc,CAAC,KAAa;QAClC,IAAI,KAAK,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;YAC9B,OAAO;gBACL,eAAe,EAAE;oBACf,MAAM,EAAE;wBACN,gBAAgB,EAAE,KAAK;qBACxB;iBACF;aACF,CAAC;QACJ,CAAC;aAAM,IAAI,KAAK,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;YACvC,OAAO;gBACL,WAAW,EAAE,CAAC;aACf,CAAC;QACJ,CAAC;QACD,OAAO,EAAE,CAAC;IACZ,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,OAAO;QACX,MAAM,IAAI,CAAC,YAAY,CAAC,OAAO,EAAE,CAAC;IACpC,CAAC;CACF;AAED;;;;;;;GAOG;AACH,MAAM,CAAC,KAAK,UAAU,sBAAsB,CAC1C,GAAW,EACX,MAAsB,EACtB,OAAiC;IAEjC,MAAM,MAAM,GAAG,IAAI,uBAAuB,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;IACjE,IAAI,CAAC;QACH,OAAO,MAAM,MAAM,CAAC,sBAAsB,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC;IACnE,CAAC;YAAS,CAAC;QACT,MAAM,MAAM,CAAC,OAAO,EAAE,CAAC;IACzB,CAAC;AACH,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@purepageio/fetch-engines",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.8.0",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"description": "A collection of configurable engines for fetching HTML content using fetch or Playwright.",
|
|
6
6
|
"main": "dist/index.js",
|
|
@@ -12,6 +12,7 @@
|
|
|
12
12
|
],
|
|
13
13
|
"dependencies": {
|
|
14
14
|
"@ai-sdk/openai": "^2.0.30",
|
|
15
|
+
"@ai-sdk/openai-compatible": "^1.0.28",
|
|
15
16
|
"ai": "^5.0.44",
|
|
16
17
|
"axios": "^1.12.0",
|
|
17
18
|
"node-html-parser": "^7.0.1",
|
|
@@ -36,6 +37,7 @@
|
|
|
36
37
|
"@types/uuid": "^10.0.0",
|
|
37
38
|
"@typescript-eslint/eslint-plugin": "^8.29.0",
|
|
38
39
|
"@typescript-eslint/parser": "^8.29.0",
|
|
40
|
+
"dotenv": "^17.2.3",
|
|
39
41
|
"eslint": "^9.23.0",
|
|
40
42
|
"eslint-config-prettier": "^10.1.1",
|
|
41
43
|
"eslint-plugin-prettier": "^5.2.5",
|