@intuned/browser-dev 0.1.9-dev.0 → 0.1.10-dev.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/ai/extractStructuredData.js +21 -27
- package/dist/ai/tests/testCreateMatchesMapping.spec.js +216 -0
- package/dist/ai/tests/testExtractStructuredData.spec.js +346 -0
- package/dist/ai/tests/testExtractStructuredDataDomMatchingIframes.spec.js +459 -0
- package/dist/ai/tests/testExtractStructuredDataUnit.spec.js +375 -0
- package/dist/ai/tests/testMatching.spec.js +342 -0
- package/dist/ai/tests/testValidateMatchesMapping.spec.js +265 -0
- package/dist/common/extendedTest.js +38 -30
- package/dist/common/frame_utils/frameTree.js +116 -0
- package/dist/common/frame_utils/getContentWithNestedIframes.js +13 -0
- package/dist/common/frame_utils/index.js +95 -0
- package/dist/common/frame_utils/stitchIframe.js +105 -0
- package/dist/{helpers → common}/frame_utils/tests/testFindAllIframes.spec.js +24 -15
- package/dist/common/frame_utils/tests/testGetContentWithNestedIframes.spec.js +241 -0
- package/dist/common/frame_utils/utils.js +91 -0
- package/dist/common/getSimplifiedHtml.js +20 -20
- package/dist/common/matching/matching.js +91 -16
- package/dist/common/tests/matching.test.js +225 -0
- package/dist/common/tests/testGetSimplifiedHtml.spec.js +324 -0
- package/dist/helpers/extractMarkdown.js +16 -7
- package/dist/helpers/tests/testExtractMarkdown.spec.js +29 -0
- package/dist/helpers/waitForDomSettled.js +4 -4
- package/dist/types/intuned-runtime.d.ts +6 -32
- package/package.json +1 -1
- package/dist/helpers/frame_utils/constants.js +0 -8
- package/dist/helpers/frame_utils/findAllIframes.js +0 -82
- package/dist/helpers/frame_utils/index.js +0 -44
- /package/dist/{helpers → common}/frame_utils/checkFrameAllowsAsyncScripts.js +0 -0
- /package/dist/{helpers → common}/frame_utils/getContainerFrame.js +0 -0
|
@@ -9,41 +9,15 @@ declare module "@intuned/runtime" {
|
|
|
9
9
|
password: string;
|
|
10
10
|
}
|
|
11
11
|
|
|
12
|
-
export type WithPlaywrightContextWrappedFunction<R> = (
|
|
13
|
-
context: BrowserContext,
|
|
14
|
-
page: Page
|
|
15
|
-
) => Promise<any>;
|
|
16
|
-
|
|
17
12
|
export function getExecutionContext(): any;
|
|
18
13
|
export function extendTimeout(): void;
|
|
19
|
-
export function runWithContext<
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
14
|
+
export function runWithContext<R, TArgs extends any[]>(
|
|
15
|
+
contextData: any,
|
|
16
|
+
callback: (...args: TArgs) => R,
|
|
17
|
+
...args: TArgs
|
|
18
|
+
): R;
|
|
23
19
|
export function getDownloadDirectoryPath(): string;
|
|
24
20
|
|
|
25
|
-
export function withPlaywrightContext<R>(
|
|
26
|
-
options: {
|
|
27
|
-
proxy?: Proxy;
|
|
28
|
-
headless: boolean;
|
|
29
|
-
downloadsPath: string;
|
|
30
|
-
importFunction?: any;
|
|
31
|
-
apiName?: string;
|
|
32
|
-
apiParameters?: any;
|
|
33
|
-
},
|
|
34
|
-
fn: WithPlaywrightContextWrappedFunction<R>
|
|
35
|
-
): Promise<any>;
|
|
36
|
-
|
|
37
|
-
export function withPlaywrightContext<R>(
|
|
38
|
-
options: {
|
|
39
|
-
cdpAddress: string;
|
|
40
|
-
importFunction?: any;
|
|
41
|
-
apiName?: string;
|
|
42
|
-
apiParameters?: any;
|
|
43
|
-
},
|
|
44
|
-
fn: WithPlaywrightContextWrappedFunction<R>
|
|
45
|
-
): Promise<any>;
|
|
46
|
-
|
|
47
21
|
// Add other exports from @intuned/runtime if needed
|
|
48
22
|
}
|
|
49
23
|
|
|
@@ -59,6 +33,6 @@ declare module "@intuned/runtime/dist/common/jwtTokenManager" {
|
|
|
59
33
|
|
|
60
34
|
export function callBackendFunctionWithToken<T = any>(
|
|
61
35
|
functionName: string,
|
|
62
|
-
params?: any
|
|
36
|
+
params?: any,
|
|
63
37
|
): Promise<T>;
|
|
64
38
|
}
|
package/package.json
CHANGED
|
@@ -1,8 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
|
|
3
|
-
Object.defineProperty(exports, "__esModule", {
|
|
4
|
-
value: true
|
|
5
|
-
});
|
|
6
|
-
exports.IFRAME_TAGS = exports.ALL_IFRAMES_CSS_SELECTOR = void 0;
|
|
7
|
-
const IFRAME_TAGS = exports.IFRAME_TAGS = ["iframe", "frame"];
|
|
8
|
-
const ALL_IFRAMES_CSS_SELECTOR = exports.ALL_IFRAMES_CSS_SELECTOR = IFRAME_TAGS.join(", ");
|
|
@@ -1,82 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
|
|
3
|
-
Object.defineProperty(exports, "__esModule", {
|
|
4
|
-
value: true
|
|
5
|
-
});
|
|
6
|
-
exports.findAllIframes = findAllIframes;
|
|
7
|
-
exports.findAllIframesList = findAllIframesList;
|
|
8
|
-
var _Logger = require("../../common/Logger");
|
|
9
|
-
var _checkFrameAllowsAsyncScripts = require("./checkFrameAllowsAsyncScripts");
|
|
10
|
-
var _constants = require("./constants");
|
|
11
|
-
async function findAllIframes(root, iframeTimeoutMs = 10000) {
|
|
12
|
-
const processed = new Set();
|
|
13
|
-
return await processFrameRecursive(root, processed, iframeTimeoutMs);
|
|
14
|
-
}
|
|
15
|
-
async function findAllIframesList(root, iframeTimeoutMs = 10000) {
|
|
16
|
-
const iframeNodes = await findAllIframes(root, iframeTimeoutMs);
|
|
17
|
-
return flattenIframeTree(iframeNodes);
|
|
18
|
-
}
|
|
19
|
-
async function processFrameRecursive(root, processedRoots, iframeTimeoutMs) {
|
|
20
|
-
if (processedRoots.has(root)) {
|
|
21
|
-
return [];
|
|
22
|
-
}
|
|
23
|
-
processedRoots.add(root);
|
|
24
|
-
const iframeNodes = [];
|
|
25
|
-
try {
|
|
26
|
-
const iframeLocator = root.locator(_constants.ALL_IFRAMES_CSS_SELECTOR);
|
|
27
|
-
let iframeCount;
|
|
28
|
-
try {
|
|
29
|
-
iframeCount = await Promise.race([iframeLocator.count(), new Promise((_, reject) => setTimeout(() => reject(new Error("Timeout")), iframeTimeoutMs))]);
|
|
30
|
-
} catch (error) {
|
|
31
|
-
_Logger.logger.error("Timeout counting iframes in context, skipping");
|
|
32
|
-
return [];
|
|
33
|
-
}
|
|
34
|
-
for (let i = 0; i < iframeCount; i++) {
|
|
35
|
-
try {
|
|
36
|
-
const processSingleIframe = async index => {
|
|
37
|
-
const iframeElementLocator = iframeLocator.nth(index);
|
|
38
|
-
const iframeElement = await iframeElementLocator.elementHandle();
|
|
39
|
-
if (!iframeElement) {
|
|
40
|
-
_Logger.logger.error(`Could not get element handle for iframe: ${iframeElement}`);
|
|
41
|
-
return null;
|
|
42
|
-
}
|
|
43
|
-
const contentFrame = await iframeElement.contentFrame();
|
|
44
|
-
if (!contentFrame) {
|
|
45
|
-
_Logger.logger.error(`Could not access content_frame for iframe: ${iframeElement}`);
|
|
46
|
-
return null;
|
|
47
|
-
}
|
|
48
|
-
const allowsAsyncScripts = await (0, _checkFrameAllowsAsyncScripts.checkFrameAllowsAsyncScripts)(iframeElement);
|
|
49
|
-
const nestedIframes = await processFrameRecursive(contentFrame, processedRoots, iframeTimeoutMs);
|
|
50
|
-
return {
|
|
51
|
-
frame: contentFrame,
|
|
52
|
-
nestedIframes,
|
|
53
|
-
allowsAsyncScripts
|
|
54
|
-
};
|
|
55
|
-
};
|
|
56
|
-
const iframeNode = await Promise.race([processSingleIframe(i), new Promise((_, reject) => setTimeout(() => reject(new Error("Timeout")), iframeTimeoutMs))]);
|
|
57
|
-
if (iframeNode !== null) {
|
|
58
|
-
iframeNodes.push(iframeNode);
|
|
59
|
-
}
|
|
60
|
-
} catch (error) {
|
|
61
|
-
_Logger.logger.error(`Timeout processing iframe ${i} in context, skipping`);
|
|
62
|
-
continue;
|
|
63
|
-
}
|
|
64
|
-
}
|
|
65
|
-
} catch (error) {
|
|
66
|
-
_Logger.logger.error(`Error processing frames in context: ${error}`);
|
|
67
|
-
}
|
|
68
|
-
return iframeNodes;
|
|
69
|
-
}
|
|
70
|
-
function flattenIframeTree(iframeNodes) {
|
|
71
|
-
const flattened = [];
|
|
72
|
-
function flattenRecursive(nodes) {
|
|
73
|
-
for (const node of nodes) {
|
|
74
|
-
flattened.push(node);
|
|
75
|
-
if (node.nestedIframes && node.nestedIframes.length > 0) {
|
|
76
|
-
flattenRecursive(node.nestedIframes);
|
|
77
|
-
}
|
|
78
|
-
}
|
|
79
|
-
}
|
|
80
|
-
flattenRecursive(iframeNodes);
|
|
81
|
-
return flattened;
|
|
82
|
-
}
|
|
@@ -1,44 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
|
|
3
|
-
Object.defineProperty(exports, "__esModule", {
|
|
4
|
-
value: true
|
|
5
|
-
});
|
|
6
|
-
Object.defineProperty(exports, "ALL_IFRAMES_CSS_SELECTOR", {
|
|
7
|
-
enumerable: true,
|
|
8
|
-
get: function () {
|
|
9
|
-
return _constants.ALL_IFRAMES_CSS_SELECTOR;
|
|
10
|
-
}
|
|
11
|
-
});
|
|
12
|
-
Object.defineProperty(exports, "IFRAME_TAGS", {
|
|
13
|
-
enumerable: true,
|
|
14
|
-
get: function () {
|
|
15
|
-
return _constants.IFRAME_TAGS;
|
|
16
|
-
}
|
|
17
|
-
});
|
|
18
|
-
Object.defineProperty(exports, "IframeNode", {
|
|
19
|
-
enumerable: true,
|
|
20
|
-
get: function () {
|
|
21
|
-
return _findAllIframes.IframeNode;
|
|
22
|
-
}
|
|
23
|
-
});
|
|
24
|
-
Object.defineProperty(exports, "findAllIframes", {
|
|
25
|
-
enumerable: true,
|
|
26
|
-
get: function () {
|
|
27
|
-
return _findAllIframes.findAllIframes;
|
|
28
|
-
}
|
|
29
|
-
});
|
|
30
|
-
Object.defineProperty(exports, "findAllIframesList", {
|
|
31
|
-
enumerable: true,
|
|
32
|
-
get: function () {
|
|
33
|
-
return _findAllIframes.findAllIframesList;
|
|
34
|
-
}
|
|
35
|
-
});
|
|
36
|
-
Object.defineProperty(exports, "getContainerFrame", {
|
|
37
|
-
enumerable: true,
|
|
38
|
-
get: function () {
|
|
39
|
-
return _getContainerFrame.getContainerFrame;
|
|
40
|
-
}
|
|
41
|
-
});
|
|
42
|
-
var _findAllIframes = require("./findAllIframes");
|
|
43
|
-
var _getContainerFrame = require("./getContainerFrame");
|
|
44
|
-
var _constants = require("./constants");
|
|
File without changes
|
|
File without changes
|