Dhalang 0.7.2 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/Dhalang/version.rb +1 -1
- data/lib/js/dhalang.js +23 -15
- data/lib/js/html-scraper.js +6 -6
- data/lib/js/pdf-generator.js +6 -6
- data/lib/js/screenshot-generator.js +5 -5
- data/package-lock.json +134 -1028
- data/package.json +2 -1
- metadata +2 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: ecb6dcd4c556f779b1a895ed19257904c10d4e5b34aa8574ffe4c9ac7f73f3e0
|
|
4
|
+
data.tar.gz: e76528a10c6c73d9729285161c04e5008dd3d4ea83e4891f1b4e798d6870f0c5
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 634e1fc5e6a48a8ca4b3fa58673cc11bbe0b2780bb061a340c45d57f00b0a70a435c04cc74bf047e9df96cf7a1b0cbfd5e5daa7a206db0ff8fc7abe63ca49554
|
|
7
|
+
data.tar.gz: 1cdd94b326c1374a4a1c263c980cd6ee33f1b694388c455d1219a2d80d10fdba15ea025928bb3b88b218072e40f2c8590118532b6573076cf0204c18a52d313f
|
data/lib/Dhalang/version.rb
CHANGED
data/lib/js/dhalang.js
CHANGED
|
@@ -1,10 +1,13 @@
|
|
|
1
|
-
|
|
1
|
+
import fs from 'node:fs';
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
import { createRequire } from "node:module";
|
|
4
|
+
import { pathToFileURL } from "node:url";
|
|
2
5
|
|
|
3
6
|
/**
|
|
4
7
|
* @typedef {Object} Configuration
|
|
5
8
|
* @property {string} webPageUrl - The url of the webpage to visit.
|
|
6
9
|
* @property {string} tempFilePath - The path of the tempfile to write the screenshot/pdf to.
|
|
7
|
-
* @property {string}
|
|
10
|
+
* @property {string} puppeteerPath - The path of the Puppeteer module.
|
|
8
11
|
* @property {string} imageType - The type of image to save ( undefined for pdfgenerator ).
|
|
9
12
|
* @property {UserOptions} userOptions - User defined and default parameters to use when navigating to pages.
|
|
10
13
|
* @property {Object} pdfOptions - User defined and default parameters to use when creating PDFs. Note: Do not use directly, rather use {@link getConfiguredPdfOptions}.
|
|
@@ -43,7 +46,7 @@ const fs = require('fs')
|
|
|
43
46
|
* @returns {Configuration}
|
|
44
47
|
* The configuration object.
|
|
45
48
|
*/
|
|
46
|
-
|
|
49
|
+
export function getConfiguration() {
|
|
47
50
|
return JSON.parse(process.argv[2])
|
|
48
51
|
}
|
|
49
52
|
|
|
@@ -53,10 +56,15 @@ exports.getConfiguration = function () {
|
|
|
53
56
|
* @returns {Promise<Object>}
|
|
54
57
|
* The launched instance of Puppeteer.
|
|
55
58
|
*/
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
59
|
+
export async function launchPuppeteer(configuration){
|
|
60
|
+
const require = createRequire(
|
|
61
|
+
path.join(configuration.puppeteerPath, "package.json")
|
|
62
|
+
);
|
|
63
|
+
const resolved = require.resolve("puppeteer");
|
|
64
|
+
const puppeteer = await import(pathToFileURL(resolved).href);
|
|
65
|
+
const launchArgs = ['--no-sandbox', '--disable-setuid-sandbox']
|
|
66
|
+
.concat(configuration.userOptions.chromeOptions)
|
|
67
|
+
.filter((item, index, self) => self.indexOf(item) === index);
|
|
60
68
|
|
|
61
69
|
if (configuration.userOptions['browserWebsocketUrl'] !== "") {
|
|
62
70
|
return await puppeteer.connect( {
|
|
@@ -76,7 +84,7 @@ exports.launchPuppeteer = async function (configuration) {
|
|
|
76
84
|
* @param {Object} page - The Puppeteer page object to configure.
|
|
77
85
|
* @param {UserOptions} userOptions - The user options to use.
|
|
78
86
|
*/
|
|
79
|
-
|
|
87
|
+
export async function configure(page, userOptions) {
|
|
80
88
|
if (userOptions.userAgent !== "") {
|
|
81
89
|
await page.setUserAgent(userOptions.userAgent)
|
|
82
90
|
}
|
|
@@ -95,16 +103,16 @@ exports.configure = async function (page, userOptions) {
|
|
|
95
103
|
* @param {Object} page - The Puppeteer page object to use for navigation.
|
|
96
104
|
* @param {Configuration} configuration - The configuration to use.
|
|
97
105
|
*/
|
|
98
|
-
|
|
106
|
+
export async function navigate(page, configuration) {
|
|
99
107
|
const navigationWaitForSelector = configuration.userOptions.navigationWaitForSelector;
|
|
100
108
|
const navigationWaitForXPath = configuration.userOptions.navigationWaitForXPath;
|
|
101
109
|
|
|
102
|
-
await page.goto(configuration.webPageUrl,
|
|
110
|
+
await page.goto(configuration.webPageUrl, getNavigationParameters(configuration));
|
|
103
111
|
|
|
104
112
|
if (navigationWaitForSelector !== "") {
|
|
105
|
-
await page.waitForSelector(navigationWaitForSelector,
|
|
113
|
+
await page.waitForSelector(navigationWaitForSelector, getWaitingParameters(configuration));
|
|
106
114
|
} else if (navigationWaitForXPath !== "") {
|
|
107
|
-
await page.waitForXPath(navigationWaitForXPath,
|
|
115
|
+
await page.waitForXPath(navigationWaitForXPath, getWaitingParameters(configuration));
|
|
108
116
|
} else {
|
|
109
117
|
await new Promise(r => setTimeout(r, 250));
|
|
110
118
|
}
|
|
@@ -116,7 +124,7 @@ exports.navigate = async function (page, configuration) {
|
|
|
116
124
|
* @param {UserOptions} configuration - The configuration to use.
|
|
117
125
|
* @returns {Object} - pdfOptions
|
|
118
126
|
*/
|
|
119
|
-
|
|
127
|
+
export async function getConfiguredPdfOptions(page, configuration) {
|
|
120
128
|
const pdfOptions = configuration.pdfOptions
|
|
121
129
|
|
|
122
130
|
if (pdfOptions['headerTemplateFile'] !== '') {
|
|
@@ -155,7 +163,7 @@ exports.getConfiguredPdfOptions = async function (page, configuration) {
|
|
|
155
163
|
* @returns {NavigationParameters}
|
|
156
164
|
* The extracted navigation parameters.
|
|
157
165
|
*/
|
|
158
|
-
|
|
166
|
+
export function getNavigationParameters(configuration) {
|
|
159
167
|
return {
|
|
160
168
|
timeout: configuration.userOptions.navigationTimeout,
|
|
161
169
|
waitUntil: configuration.userOptions.navigationWaitUntil
|
|
@@ -169,7 +177,7 @@ exports.getNavigationParameters = function (configuration) {
|
|
|
169
177
|
* @returns {WaitingParameters}
|
|
170
178
|
* The extracted waiting parameters.
|
|
171
179
|
*/
|
|
172
|
-
|
|
180
|
+
export function getWaitingParameters(configuration) {
|
|
173
181
|
return {
|
|
174
182
|
timeout: configuration.userOptions.navigationTimeout
|
|
175
183
|
}
|
data/lib/js/html-scraper.js
CHANGED
|
@@ -1,17 +1,17 @@
|
|
|
1
1
|
'use strict';
|
|
2
|
-
|
|
3
|
-
|
|
2
|
+
import {getConfiguration, launchPuppeteer, configure, navigate} from './dhalang.js';
|
|
3
|
+
import fs from 'node:fs';
|
|
4
4
|
|
|
5
5
|
const scrapeHtml = async () => {
|
|
6
|
-
const configuration =
|
|
6
|
+
const configuration = getConfiguration();
|
|
7
7
|
|
|
8
8
|
let browser;
|
|
9
9
|
let page;
|
|
10
10
|
try {
|
|
11
|
-
browser = await
|
|
11
|
+
browser = await launchPuppeteer(configuration);
|
|
12
12
|
page = await browser.newPage();
|
|
13
|
-
await
|
|
14
|
-
await
|
|
13
|
+
await configure(page, configuration.userOptions);
|
|
14
|
+
await navigate(page, configuration);
|
|
15
15
|
const html = await page.content();
|
|
16
16
|
fs.writeFileSync(configuration.tempFilePath, html);
|
|
17
17
|
} catch (error) {
|
data/lib/js/pdf-generator.js
CHANGED
|
@@ -1,17 +1,17 @@
|
|
|
1
1
|
'use strict';
|
|
2
|
-
|
|
2
|
+
import {getConfiguration, getConfiguredPdfOptions, launchPuppeteer, configure, navigate} from './dhalang.js';
|
|
3
3
|
|
|
4
4
|
const createPdf = async () => {
|
|
5
|
-
const configuration =
|
|
5
|
+
const configuration = getConfiguration();
|
|
6
6
|
|
|
7
7
|
let browser;
|
|
8
8
|
let page;
|
|
9
9
|
try {
|
|
10
|
-
browser = await
|
|
10
|
+
browser = await launchPuppeteer(configuration);
|
|
11
11
|
page = await browser.newPage();
|
|
12
|
-
await
|
|
13
|
-
await
|
|
14
|
-
const pdfOptions = await
|
|
12
|
+
await configure(page, configuration.userOptions);
|
|
13
|
+
await navigate(page, configuration);
|
|
14
|
+
const pdfOptions = await getConfiguredPdfOptions(page, configuration);
|
|
15
15
|
await page.pdf({
|
|
16
16
|
...{
|
|
17
17
|
path: configuration.tempFilePath
|
|
@@ -1,16 +1,16 @@
|
|
|
1
1
|
'use strict';
|
|
2
|
-
|
|
2
|
+
import {getConfiguration, launchPuppeteer, configure, navigate} from './dhalang.js';
|
|
3
3
|
|
|
4
4
|
const createScreenshot = async () => {
|
|
5
|
-
const configuration =
|
|
5
|
+
const configuration = getConfiguration();
|
|
6
6
|
|
|
7
7
|
let browser;
|
|
8
8
|
let page;
|
|
9
9
|
try {
|
|
10
|
-
browser = await
|
|
10
|
+
browser = await launchPuppeteer(configuration);
|
|
11
11
|
page = await browser.newPage();
|
|
12
|
-
await
|
|
13
|
-
await
|
|
12
|
+
await configure(page, configuration.userOptions);
|
|
13
|
+
await navigate(page, configuration);
|
|
14
14
|
|
|
15
15
|
await page.screenshot({
|
|
16
16
|
...{
|