@intuned/browser-dev 2.2.3-unify-sdks.20
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.babelrc +21 -0
- package/.eslintignore +10 -0
- package/.eslintrc.js +39 -0
- package/LICENSE +43 -0
- package/dist/ai-extractors/AnthropicClient/index.js +23 -0
- package/dist/ai-extractors/export.d.js +5 -0
- package/dist/ai-extractors/export.d.ts +422 -0
- package/dist/ai-extractors/extractStructuredData.js +79 -0
- package/dist/ai-extractors/extractionHelpers/extractStructuredDataUsingAi/constants.js +7 -0
- package/dist/ai-extractors/extractionHelpers/extractStructuredDataUsingAi/errors.js +42 -0
- package/dist/ai-extractors/extractionHelpers/extractStructuredDataUsingAi/extractStructuredDataUsingClaude.js +149 -0
- package/dist/ai-extractors/extractionHelpers/extractStructuredDataUsingAi/extractStructuredDataUsingGoogle.js +37 -0
- package/dist/ai-extractors/extractionHelpers/extractStructuredDataUsingAi/extractStructuredDataUsingOpenAi.js +144 -0
- package/dist/ai-extractors/extractionHelpers/extractStructuredDataUsingAi/extractStrucutredDataUsingAiInstance.js +123 -0
- package/dist/ai-extractors/extractionHelpers/extractStructuredDataUsingAi/index.js +55 -0
- package/dist/ai-extractors/extractionHelpers/extractStructuredDataUsingAi/isItemTableHeaderOrFooter.js +96 -0
- package/dist/ai-extractors/extractionHelpers/extractStructuredDataUsingAi/screenshotHelpers.js +55 -0
- package/dist/ai-extractors/extractionHelpers/extractStructuredDataUsingAi/types.js +5 -0
- package/dist/ai-extractors/extractionHelpers/extractStructuredDataUsingAi/utils.js +53 -0
- package/dist/ai-extractors/extractionHelpers/types.js +5 -0
- package/dist/ai-extractors/fileExtractors.js +176 -0
- package/dist/ai-extractors/index.js +31 -0
- package/dist/ai-extractors/jsonSchema.d.js +5 -0
- package/dist/ai-extractors/jsonSchema.d.ts +49 -0
- package/dist/ai-extractors/openAiClients/index.js +23 -0
- package/dist/ai-extractors/validators.js +239 -0
- package/dist/browser/ai/export.d.js +3 -0
- package/dist/browser/ai/export.d.ts +587 -0
- package/dist/browser/ai/extractMarkdown.js +15 -0
- package/dist/browser/ai/extractStructuredData.js +231 -0
- package/dist/browser/ai/extractStructuredDataUsingAi.js +140 -0
- package/dist/browser/ai/extractionHelpers/screenshotHelpers.js +55 -0
- package/dist/browser/ai/extractionHelpers/validateSchema.js +148 -0
- package/dist/browser/ai/index.d.ts +587 -0
- package/dist/browser/ai/index.js +19 -0
- package/dist/browser/ai/isPageLoaded.js +67 -0
- package/dist/browser/ai/prompt.js +39 -0
- package/dist/browser/ai/tests/testCheckAllTypesAreStrings.spec.js +143 -0
- package/dist/browser/ai/tests/testExtractStructuredData.spec.js +622 -0
- package/dist/browser/ai/tools/index.js +48 -0
- package/dist/browser/ai/types/errors.js +67 -0
- package/dist/browser/ai/types/models.js +45 -0
- package/dist/browser/ai/types/types.js +48 -0
- package/dist/browser/ai/validators.js +136 -0
- package/dist/common/Logger/index.js +60 -0
- package/dist/common/Logger/types.js +5 -0
- package/dist/common/SdkError.js +50 -0
- package/dist/common/aiModelsValidations.js +50 -0
- package/dist/common/browser_scripts.js +2596 -0
- package/dist/common/ensureBrowserScripts.js +17 -0
- package/dist/common/environmentVariables.js +16 -0
- package/dist/common/eventTracking/getAiTrackingHeaders.js +31 -0
- package/dist/common/eventTracking/getFileTrackingHeaders.js +23 -0
- package/dist/common/extendedTest.js +148 -0
- package/dist/common/extractionHelpers.js +19 -0
- package/dist/common/formatZodError.js +18 -0
- package/dist/common/fuzzySearch/fuzzySearch.test.js +250 -0
- package/dist/common/fuzzySearch/levenshtein-search.js +298 -0
- package/dist/common/fuzzySearch/utils.js +23 -0
- package/dist/common/getModelProvider.js +18 -0
- package/dist/common/getSimplifiedHtml.js +122 -0
- package/dist/common/hashObject.js +32 -0
- package/dist/common/html2markdown/convertElementToMarkdown.js +469 -0
- package/dist/common/html2markdown/index.js +19 -0
- package/dist/common/jwtTokenManager.js +18 -0
- package/dist/common/loadRuntime.js +16 -0
- package/dist/common/locatorHelpers.js +41 -0
- package/dist/common/matching/collectStrings.js +32 -0
- package/dist/common/matching/levenshtein.js +40 -0
- package/dist/common/matching/matching.js +317 -0
- package/dist/common/matching/types.js +1 -0
- package/dist/common/noEmpty.js +9 -0
- package/dist/common/saveSnapshotWithExamples.js +60 -0
- package/dist/common/tests/testEnsureBrowserScript.spec.js +31 -0
- package/dist/common/xpathMapping.js +107 -0
- package/dist/helpers/downloadFile.js +125 -0
- package/dist/helpers/export.d.js +1 -0
- package/dist/helpers/export.d.ts +1294 -0
- package/dist/helpers/extractMarkdown.js +35 -0
- package/dist/helpers/filterEmptyValues.js +54 -0
- package/dist/helpers/gotoUrl.js +93 -0
- package/dist/helpers/index.d.ts +1294 -0
- package/dist/helpers/index.js +115 -0
- package/dist/helpers/processDate.js +25 -0
- package/dist/helpers/resolveUrl.js +63 -0
- package/dist/helpers/sanitizeHtml.js +73 -0
- package/dist/helpers/saveFileToS3.js +46 -0
- package/dist/helpers/scrollToLoadContent.js +50 -0
- package/dist/helpers/tests/extendedTest.js +130 -0
- package/dist/helpers/tests/testDownloadFile.spec.js +197 -0
- package/dist/helpers/tests/testFilterEmptyValues.spec.js +151 -0
- package/dist/helpers/tests/testGoToUrl.spec.js +37 -0
- package/dist/helpers/tests/testIsPageLoaded.spec.js +285 -0
- package/dist/helpers/tests/testProcessDate.spec.js +13 -0
- package/dist/helpers/tests/testResolveUrl.spec.js +341 -0
- package/dist/helpers/tests/testSanitizeHtml.spec.js +330 -0
- package/dist/helpers/tests/testSimplifyHtml.spec.js +251 -0
- package/dist/helpers/tests/testValidateDataUsingSchema.spec.js +380 -0
- package/dist/helpers/tests/testWaitForDomSettled.spec.js +169 -0
- package/dist/helpers/tests/testWaitForNetworkIdle.spec.js +115 -0
- package/dist/helpers/types/Attachment.js +81 -0
- package/dist/helpers/types/CustomTypeRegistry.js +48 -0
- package/dist/helpers/types/RunEnvironment.js +18 -0
- package/dist/helpers/types/ValidationError.js +17 -0
- package/dist/helpers/types/index.js +51 -0
- package/dist/helpers/uploadFileToS3.js +153 -0
- package/dist/helpers/utils/getS3Client.js +21 -0
- package/dist/helpers/utils/index.js +73 -0
- package/dist/helpers/utils/isDownload.js +10 -0
- package/dist/helpers/utils/isGenerateCodeMode.js +9 -0
- package/dist/helpers/utils/isLocator.js +9 -0
- package/dist/helpers/utils/jwtTokenManager.js +18 -0
- package/dist/helpers/validateDataUsingSchema.js +119 -0
- package/dist/helpers/waitForDomSettled.js +182 -0
- package/dist/helpers/waitForNetworkIdle.js +191 -0
- package/dist/index.d.js +82 -0
- package/dist/index.d.ts +11 -0
- package/dist/index.js +84 -0
- package/dist/intunedServices/ApiGateway/aiApiGateway.js +87 -0
- package/dist/intunedServices/ApiGateway/factory.js +13 -0
- package/dist/intunedServices/ApiGateway/providers/Anthropic.js +26 -0
- package/dist/intunedServices/ApiGateway/providers/Gemini.js +29 -0
- package/dist/intunedServices/ApiGateway/providers/OpenAI.js +29 -0
- package/dist/intunedServices/ApiGateway/tests/testApiGateway.spec.js +221 -0
- package/dist/intunedServices/ApiGateway/types.js +11 -0
- package/dist/intunedServices/cache/cache.js +61 -0
- package/dist/intunedServices/cache/index.js +12 -0
- package/dist/intunedServices/cache/tests/testCache.spec.js +117 -0
- package/dist/optimized-extractors/common/buildExamplesPrompt.js +12 -0
- package/dist/optimized-extractors/common/buildImagesFromPage.js +55 -0
- package/dist/optimized-extractors/common/extractStructuredDataUsingClaude.js +149 -0
- package/dist/optimized-extractors/common/extractStructuredDataUsingGoogle.js +37 -0
- package/dist/optimized-extractors/common/extractStructuredDataUsingOpenAi.js +145 -0
- package/dist/optimized-extractors/common/extractStrucutredDataUsingAiInstance.js +122 -0
- package/dist/optimized-extractors/common/findTableHeaders.js +175 -0
- package/dist/optimized-extractors/common/index.js +55 -0
- package/dist/optimized-extractors/common/isTableHeaderOrFooter.js +97 -0
- package/dist/optimized-extractors/common/matching/matching.js +212 -0
- package/dist/optimized-extractors/common/matching/matching.test.js +655 -0
- package/dist/optimized-extractors/common/matching/types.js +18 -0
- package/dist/optimized-extractors/common/matching/utils.js +184 -0
- package/dist/optimized-extractors/common/utils.js +58 -0
- package/dist/optimized-extractors/export.d.js +5 -0
- package/dist/optimized-extractors/export.d.ts +397 -0
- package/dist/optimized-extractors/extractArray.js +120 -0
- package/dist/optimized-extractors/extractObject.js +104 -0
- package/dist/optimized-extractors/index.d.ts +397 -0
- package/dist/optimized-extractors/index.js +31 -0
- package/dist/optimized-extractors/listExtractionHelpers/__tests__/dynamicListExtractor.spec.js +312 -0
- package/dist/optimized-extractors/listExtractionHelpers/__tests__/findSetOfXpathsToCreateAnArrayExtractor.test.js +22 -0
- package/dist/optimized-extractors/listExtractionHelpers/__tests__/getContainerElement.test.js +21 -0
- package/dist/optimized-extractors/listExtractionHelpers/__tests__/partOfSameArrayXpath.test.js +42 -0
- package/dist/optimized-extractors/listExtractionHelpers/__tests__/verifyThatAllXpathsArePartOfSameArray.test.js +9 -0
- package/dist/optimized-extractors/listExtractionHelpers/dynamicListExtractor.js +152 -0
- package/dist/optimized-extractors/listExtractionHelpers/errors.js +46 -0
- package/dist/optimized-extractors/listExtractionHelpers/getListMatches.js +14 -0
- package/dist/optimized-extractors/listExtractionHelpers/runAiExtraction.js +240 -0
- package/dist/optimized-extractors/listExtractionHelpers/typesAndSchema.js +5 -0
- package/dist/optimized-extractors/listExtractionHelpers/utils/extractPropertiesUsingGPTFromArray.js +277 -0
- package/dist/optimized-extractors/listExtractionHelpers/utils/extractStructuredListUsingAi.js +44 -0
- package/dist/optimized-extractors/listExtractionHelpers/utils/getListContainerXpath.js +94 -0
- package/dist/optimized-extractors/listExtractionHelpers/utils/getRelativeContainerXpathSelector.js +20 -0
- package/dist/optimized-extractors/listExtractionHelpers/utils/getSimplifiedHtmlPerListItem.js +21 -0
- package/dist/optimized-extractors/listExtractionHelpers/utils/tablesUtils.js +48 -0
- package/dist/optimized-extractors/listExtractionHelpers/utils/validateOptions.js +52 -0
- package/dist/optimized-extractors/models/anthropicModel.js +23 -0
- package/dist/optimized-extractors/models/openaiModel.js +23 -0
- package/dist/optimized-extractors/objectExtractionHelpers/AIExtractors.js +73 -0
- package/dist/optimized-extractors/objectExtractionHelpers/__tests__/checksumUtils.test.js +103 -0
- package/dist/optimized-extractors/objectExtractionHelpers/__tests__/testObjectExtractorFromLocator.spec.js +107 -0
- package/dist/optimized-extractors/objectExtractionHelpers/__tests__/testObjectExtractorFromPage.spec.js +107 -0
- package/dist/optimized-extractors/objectExtractionHelpers/calculateObjectExampleHash.js +28 -0
- package/dist/optimized-extractors/objectExtractionHelpers/captureSnapshot.js +26 -0
- package/dist/optimized-extractors/objectExtractionHelpers/checksumUtils.js +32 -0
- package/dist/optimized-extractors/objectExtractionHelpers/constants.js +7 -0
- package/dist/optimized-extractors/objectExtractionHelpers/dynamicObjectExtractor.js +106 -0
- package/dist/optimized-extractors/objectExtractionHelpers/errors.js +42 -0
- package/dist/optimized-extractors/objectExtractionHelpers/findDomMatches.js +54 -0
- package/dist/optimized-extractors/objectExtractionHelpers/getSimplifiedHtml.js +122 -0
- package/dist/optimized-extractors/objectExtractionHelpers/typesAndSchemas.js +5 -0
- package/dist/optimized-extractors/objectExtractionHelpers/validateDynamicObjectExtractorOptions.js +52 -0
- package/dist/optimized-extractors/types/aiModelsValidation.js +45 -0
- package/dist/optimized-extractors/types/errors.js +42 -0
- package/dist/optimized-extractors/types/jsonSchema.d.js +5 -0
- package/dist/optimized-extractors/types/jsonSchema.d.ts +50 -0
- package/dist/optimized-extractors/types/types.js +5 -0
- package/dist/optimized-extractors/validators.js +152 -0
- package/dist/vite-env.d.js +1 -0
- package/dist/vite-env.d.ts +9 -0
- package/docs.md +14 -0
- package/how-to-run-tests.md +10 -0
- package/intuned-runtime-setup.md +13 -0
- package/package.json +124 -0
- package/tsconfig.eslint.json +5 -0
- package/tsconfig.json +26 -0
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
|
|
3
|
+
Object.defineProperty(exports, "__esModule", {
|
|
4
|
+
value: true
|
|
5
|
+
});
|
|
6
|
+
exports.getMessages = getMessages;
|
|
7
|
+
function getMessages(input) {
|
|
8
|
+
const {
|
|
9
|
+
prompt,
|
|
10
|
+
content,
|
|
11
|
+
images
|
|
12
|
+
} = input;
|
|
13
|
+
const messages = [{
|
|
14
|
+
role: "system",
|
|
15
|
+
content: `You are a data analyst whose job is to extract structured data from an HTML page.
|
|
16
|
+
Please ensure that the data is extracted exactly as it appears in the HTML, without any additional formatting or alterations.
|
|
17
|
+
Extract the structured data exactly as it is in the HTML.
|
|
18
|
+
If you don't find a specific field just don't return the field.
|
|
19
|
+
Call extract_data_from_content tool with the extracted data as the argument of this tool.
|
|
20
|
+
${prompt}`
|
|
21
|
+
}];
|
|
22
|
+
const userContentParts = [{
|
|
23
|
+
type: "text",
|
|
24
|
+
text: content
|
|
25
|
+
}];
|
|
26
|
+
if (images && images.length > 0) {
|
|
27
|
+
images.forEach(imageUrl => {
|
|
28
|
+
userContentParts.push({
|
|
29
|
+
type: "image",
|
|
30
|
+
image: `data:image/png;base64,${imageUrl.toString("base64")}`
|
|
31
|
+
});
|
|
32
|
+
});
|
|
33
|
+
}
|
|
34
|
+
messages.push({
|
|
35
|
+
role: "user",
|
|
36
|
+
content: userContentParts
|
|
37
|
+
});
|
|
38
|
+
return messages;
|
|
39
|
+
}
|
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
|
|
3
|
+
var _extendedTest = require("../../../helpers/tests/extendedTest");
|
|
4
|
+
var _validators = require("../validators");
|
|
5
|
+
(0, _extendedTest.describe)("Check All Types Are Strings Tests", () => {
|
|
6
|
+
(0, _extendedTest.test)("should return true for string schema", async () => {
|
|
7
|
+
const schema = {
|
|
8
|
+
type: "string",
|
|
9
|
+
description: "A simple string field"
|
|
10
|
+
};
|
|
11
|
+
(0, _extendedTest.expect)((0, _validators.checkAllTypesAreStrings)(schema)).toBe(true);
|
|
12
|
+
});
|
|
13
|
+
(0, _extendedTest.test)("should return false for number schema", async () => {
|
|
14
|
+
const schema = {
|
|
15
|
+
type: "number",
|
|
16
|
+
description: "A number field"
|
|
17
|
+
};
|
|
18
|
+
(0, _extendedTest.expect)((0, _validators.checkAllTypesAreStrings)(schema)).toBe(false);
|
|
19
|
+
});
|
|
20
|
+
(0, _extendedTest.test)("should return false for boolean schema", async () => {
|
|
21
|
+
const schema = {
|
|
22
|
+
type: "boolean",
|
|
23
|
+
description: "A boolean field"
|
|
24
|
+
};
|
|
25
|
+
(0, _extendedTest.expect)((0, _validators.checkAllTypesAreStrings)(schema)).toBe(false);
|
|
26
|
+
});
|
|
27
|
+
(0, _extendedTest.test)("should return true for array of strings", async () => {
|
|
28
|
+
const schema = {
|
|
29
|
+
type: "array",
|
|
30
|
+
items: {
|
|
31
|
+
type: "string"
|
|
32
|
+
}
|
|
33
|
+
};
|
|
34
|
+
(0, _extendedTest.expect)((0, _validators.checkAllTypesAreStrings)(schema)).toBe(true);
|
|
35
|
+
});
|
|
36
|
+
(0, _extendedTest.test)("should return false for array of mixed types", async () => {
|
|
37
|
+
const schema = {
|
|
38
|
+
type: "array",
|
|
39
|
+
items: [{
|
|
40
|
+
type: "string"
|
|
41
|
+
}, {
|
|
42
|
+
type: "number"
|
|
43
|
+
}]
|
|
44
|
+
};
|
|
45
|
+
(0, _extendedTest.expect)((0, _validators.checkAllTypesAreStrings)(schema)).toBe(false);
|
|
46
|
+
});
|
|
47
|
+
(0, _extendedTest.test)("should return true for array with no items constraint", async () => {
|
|
48
|
+
const schema = {
|
|
49
|
+
type: "array"
|
|
50
|
+
};
|
|
51
|
+
(0, _extendedTest.expect)((0, _validators.checkAllTypesAreStrings)(schema)).toBe(true);
|
|
52
|
+
});
|
|
53
|
+
(0, _extendedTest.test)("should return true for object with all string properties", async () => {
|
|
54
|
+
const schema = {
|
|
55
|
+
type: "object",
|
|
56
|
+
properties: {
|
|
57
|
+
name: {
|
|
58
|
+
type: "string"
|
|
59
|
+
},
|
|
60
|
+
description: {
|
|
61
|
+
type: "string"
|
|
62
|
+
}
|
|
63
|
+
},
|
|
64
|
+
required: ["name", "description"]
|
|
65
|
+
};
|
|
66
|
+
(0, _extendedTest.expect)((0, _validators.checkAllTypesAreStrings)(schema)).toBe(true);
|
|
67
|
+
});
|
|
68
|
+
(0, _extendedTest.test)("should return false for object with mixed property types", async () => {
|
|
69
|
+
const schema = {
|
|
70
|
+
type: "object",
|
|
71
|
+
properties: {
|
|
72
|
+
name: {
|
|
73
|
+
type: "string"
|
|
74
|
+
},
|
|
75
|
+
age: {
|
|
76
|
+
type: "number"
|
|
77
|
+
}
|
|
78
|
+
},
|
|
79
|
+
required: ["name", "age"]
|
|
80
|
+
};
|
|
81
|
+
(0, _extendedTest.expect)((0, _validators.checkAllTypesAreStrings)(schema)).toBe(false);
|
|
82
|
+
});
|
|
83
|
+
(0, _extendedTest.test)("should return true for object with no properties", async () => {
|
|
84
|
+
const schema = {
|
|
85
|
+
type: "object"
|
|
86
|
+
};
|
|
87
|
+
(0, _extendedTest.expect)((0, _validators.checkAllTypesAreStrings)(schema)).toBe(true);
|
|
88
|
+
});
|
|
89
|
+
(0, _extendedTest.test)("should return true for nested object with all string properties", async () => {
|
|
90
|
+
const schema = {
|
|
91
|
+
type: "object",
|
|
92
|
+
properties: {
|
|
93
|
+
user: {
|
|
94
|
+
type: "object",
|
|
95
|
+
properties: {
|
|
96
|
+
firstName: {
|
|
97
|
+
type: "string"
|
|
98
|
+
},
|
|
99
|
+
lastName: {
|
|
100
|
+
type: "string"
|
|
101
|
+
}
|
|
102
|
+
},
|
|
103
|
+
required: ["firstName", "lastName"]
|
|
104
|
+
},
|
|
105
|
+
metadata: {
|
|
106
|
+
type: "object",
|
|
107
|
+
properties: {
|
|
108
|
+
tags: {
|
|
109
|
+
type: "array",
|
|
110
|
+
items: {
|
|
111
|
+
type: "string"
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
},
|
|
115
|
+
required: ["tags"]
|
|
116
|
+
}
|
|
117
|
+
},
|
|
118
|
+
required: ["user", "metadata"]
|
|
119
|
+
};
|
|
120
|
+
(0, _extendedTest.expect)((0, _validators.checkAllTypesAreStrings)(schema)).toBe(true);
|
|
121
|
+
});
|
|
122
|
+
(0, _extendedTest.test)("should return false for nested object with non-string properties", async () => {
|
|
123
|
+
const schema = {
|
|
124
|
+
type: "object",
|
|
125
|
+
properties: {
|
|
126
|
+
user: {
|
|
127
|
+
type: "object",
|
|
128
|
+
properties: {
|
|
129
|
+
firstName: {
|
|
130
|
+
type: "string"
|
|
131
|
+
},
|
|
132
|
+
age: {
|
|
133
|
+
type: "number"
|
|
134
|
+
}
|
|
135
|
+
},
|
|
136
|
+
required: ["firstName", "age"]
|
|
137
|
+
}
|
|
138
|
+
},
|
|
139
|
+
required: ["user"]
|
|
140
|
+
};
|
|
141
|
+
(0, _extendedTest.expect)((0, _validators.checkAllTypesAreStrings)(schema)).toBe(false);
|
|
142
|
+
});
|
|
143
|
+
});
|