Dhalang 0.7.1 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile.lock +7 -7
- data/README.md +5 -5
- data/lib/Dhalang/configuration.rb +2 -1
- data/lib/Dhalang/url_utils.rb +5 -4
- data/lib/Dhalang/version.rb +1 -1
- data/lib/js/dhalang.js +27 -17
- data/lib/js/html-scraper.js +6 -6
- data/lib/js/pdf-generator.js +6 -6
- data/lib/js/screenshot-generator.js +5 -5
- data/package-lock.json +165 -1006
- data/package.json +3 -2
- data/renovate.json +15 -0
- metadata +3 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: ecb6dcd4c556f779b1a895ed19257904c10d4e5b34aa8574ffe4c9ac7f73f3e0
|
|
4
|
+
data.tar.gz: e76528a10c6c73d9729285161c04e5008dd3d4ea83e4891f1b4e798d6870f0c5
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 634e1fc5e6a48a8ca4b3fa58673cc11bbe0b2780bb061a340c45d57f00b0a70a435c04cc74bf047e9df96cf7a1b0cbfd5e5daa7a206db0ff8fc7abe63ca49554
|
|
7
|
+
data.tar.gz: 1cdd94b326c1374a4a1c263c980cd6ee33f1b694388c455d1219a2d80d10fdba15ea025928bb3b88b218072e40f2c8590118532b6573076cf0204c18a52d313f
|
data/Gemfile.lock
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
PATH
|
|
2
2
|
remote: .
|
|
3
3
|
specs:
|
|
4
|
-
Dhalang (0.7.
|
|
4
|
+
Dhalang (0.7.2)
|
|
5
5
|
|
|
6
6
|
GEM
|
|
7
7
|
remote: https://rubygems.org/
|
|
8
8
|
specs:
|
|
9
|
-
Ascii85 (1.1.
|
|
9
|
+
Ascii85 (1.1.1)
|
|
10
10
|
afm (0.2.2)
|
|
11
|
-
bigdecimal (3.1.
|
|
11
|
+
bigdecimal (3.1.8)
|
|
12
12
|
diff-lcs (1.5.1)
|
|
13
13
|
fastimage (2.2.7)
|
|
14
14
|
hashery (2.1.2)
|
|
@@ -23,15 +23,15 @@ GEM
|
|
|
23
23
|
rspec-core (~> 3.13.0)
|
|
24
24
|
rspec-expectations (~> 3.13.0)
|
|
25
25
|
rspec-mocks (~> 3.13.0)
|
|
26
|
-
rspec-core (3.13.
|
|
26
|
+
rspec-core (3.13.2)
|
|
27
27
|
rspec-support (~> 3.13.0)
|
|
28
|
-
rspec-expectations (3.13.
|
|
28
|
+
rspec-expectations (3.13.3)
|
|
29
29
|
diff-lcs (>= 1.2.0, < 2.0)
|
|
30
30
|
rspec-support (~> 3.13.0)
|
|
31
|
-
rspec-mocks (3.13.
|
|
31
|
+
rspec-mocks (3.13.2)
|
|
32
32
|
diff-lcs (>= 1.2.0, < 2.0)
|
|
33
33
|
rspec-support (~> 3.13.0)
|
|
34
|
-
rspec-support (3.13.
|
|
34
|
+
rspec-support (3.13.2)
|
|
35
35
|
ruby-rc4 (0.1.5)
|
|
36
36
|
ttfunk (1.8.0)
|
|
37
37
|
bigdecimal (~> 3.1)
|
data/README.md
CHANGED
|
@@ -80,10 +80,9 @@ For example to only take a screenshot of the visible part of the page:
|
|
|
80
80
|
Dhalang::Screenshot.get_from_url("https://www.google.com", :webp, {fullPage: false})
|
|
81
81
|
```
|
|
82
82
|
|
|
83
|
-
A list of all possible PDF options that can be set, can be found at: https://github.com/puppeteer/puppeteer/blob/main/docs/api.md
|
|
84
|
-
|
|
85
|
-
A list of all possible screenshot options that can be set, can be found at: https://github.com/puppeteer/puppeteer/blob/main/docs/api.md#pagescreenshotoptions
|
|
83
|
+
A list of all possible PDF options that can be set, can be found at: https://github.com/puppeteer/puppeteer/blob/main/docs/api/puppeteer.pdfoptions.md
|
|
86
84
|
|
|
85
|
+
A list of all possible screenshot options that can be set, can be found at: https://github.com/puppeteer/puppeteer/blob/main/docs/api/puppeteer.screenshotoptions.md
|
|
87
86
|
> The default Puppeteer options contain the options `headerTemplate` and `footerTemplate`. Puppeteer expects these to be HTML strings. By default, the Dhalang
|
|
88
87
|
> gem passes all options as arguments in a `node ...` shell command. In case the HTML strings are too long they might surpass the maximum
|
|
89
88
|
> argument length of the host. For example, on Linux the `MAX_ARG_LEN` is 128kB. Therefore, you can also pass the headers and footers as file path using the
|
|
@@ -94,13 +93,14 @@ A list of all possible screenshot options that can be set, can be found at: http
|
|
|
94
93
|
Below table lists more configuration parameters that can be set:
|
|
95
94
|
| Key | Description | Default |
|
|
96
95
|
|--------------------|-----------------------------------------------------------------------------------------|---------------------------------|
|
|
96
|
+
| isHeadless | Indicates if Chromium should be launched headless (useful for debugging) | true |
|
|
97
|
+
| slowMo | Amount of milliseconds to slow down Puppeteer operations (useful for debugging) | 0 |
|
|
97
98
|
| browserWebsocketUrl | Websocket url of remote chromium browser to use | None |
|
|
98
99
|
| navigationTimeout | Amount of milliseconds until Puppeteer while timeout when navigating to the given page | 10000 |
|
|
99
100
|
| printToPDFTimeout | Amount of milliseconds until Puppeteer while timeout when calling Page.printToPDF | 0 (unlimited) |
|
|
100
101
|
| navigationWaitForSelector | If set, Dhalang will wait for the specified selector to appear before creating the screenshot or PDF | None |
|
|
101
102
|
| navigationWaitForXPath | If set, Dhalang will wait for the specified XPath to appear before creating the screenshot or PDF | None |
|
|
102
103
|
| userAgent | User agent to send with the request | Default Puppeteer one |
|
|
103
|
-
| isHeadless | Indicates if Chromium should be launched headless | true |
|
|
104
104
|
| isAutoHeight | When set to true the height of generated PDFs will be based on the scrollHeight property of the document body | false |
|
|
105
105
|
| viewPort | Custom viewport to use for the request | Default Puppeteer one |
|
|
106
106
|
| httpAuthenticationCredentials | Custom HTTP authentication credentials to use for the request | None |
|
|
@@ -122,4 +122,4 @@ def example_controller_method
|
|
|
122
122
|
binary_png = Dhalang::Screenshot.get_from_url("https://www.google.com", :png)
|
|
123
123
|
send_data(binary_png, filename: 'screenshotofgoogle.png', type: 'image/png')
|
|
124
124
|
end
|
|
125
|
-
```
|
|
125
|
+
```
|
data/lib/Dhalang/url_utils.rb
CHANGED
|
@@ -6,9 +6,10 @@ module Dhalang
|
|
|
6
6
|
#
|
|
7
7
|
# @param [String] url The url to validate
|
|
8
8
|
def self.validate(url)
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
9
|
+
parsed = URI.parse(url) # Raise URI::InvalidURIError on invalid URLs
|
|
10
|
+
return true if parsed.absolute?
|
|
11
|
+
|
|
12
|
+
raise URI::InvalidURIError, 'The given url was invalid, use format http://www.example.com'
|
|
12
13
|
end
|
|
13
14
|
end
|
|
14
|
-
end
|
|
15
|
+
end
|
data/lib/Dhalang/version.rb
CHANGED
data/lib/js/dhalang.js
CHANGED
|
@@ -1,10 +1,13 @@
|
|
|
1
|
-
|
|
1
|
+
import fs from 'node:fs';
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
import { createRequire } from "node:module";
|
|
4
|
+
import { pathToFileURL } from "node:url";
|
|
2
5
|
|
|
3
6
|
/**
|
|
4
7
|
* @typedef {Object} Configuration
|
|
5
8
|
* @property {string} webPageUrl - The url of the webpage to visit.
|
|
6
9
|
* @property {string} tempFilePath - The path of the tempfile to write the screenshot/pdf to.
|
|
7
|
-
* @property {string}
|
|
10
|
+
* @property {string} puppeteerPath - The path of the Puppeteer module.
|
|
8
11
|
* @property {string} imageType - The type of image to save ( undefined for pdfgenerator ).
|
|
9
12
|
* @property {UserOptions} userOptions - User defined and default parameters to use when navigating to pages.
|
|
10
13
|
* @property {Object} pdfOptions - User defined and default parameters to use when creating PDFs. Note: Do not use directly, rather use {@link getConfiguredPdfOptions}.
|
|
@@ -24,6 +27,7 @@ const fs = require('fs')
|
|
|
24
27
|
* @property {Object} viewPort - The view port to use.
|
|
25
28
|
* @property {Object} httpAuthenticationCredentials - The credentials to use for HTTP authentication.
|
|
26
29
|
* @property {boolean} isAutoHeight - The height is automatically set
|
|
30
|
+
* @property {number} slowMo - Amount of milliseconds to slow down Puppeteer operations.
|
|
27
31
|
*/
|
|
28
32
|
|
|
29
33
|
/**
|
|
@@ -42,7 +46,7 @@ const fs = require('fs')
|
|
|
42
46
|
* @returns {Configuration}
|
|
43
47
|
* The configuration object.
|
|
44
48
|
*/
|
|
45
|
-
|
|
49
|
+
export function getConfiguration() {
|
|
46
50
|
return JSON.parse(process.argv[2])
|
|
47
51
|
}
|
|
48
52
|
|
|
@@ -52,10 +56,15 @@ exports.getConfiguration = function () {
|
|
|
52
56
|
* @returns {Promise<Object>}
|
|
53
57
|
* The launched instance of Puppeteer.
|
|
54
58
|
*/
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
+
export async function launchPuppeteer(configuration){
|
|
60
|
+
const require = createRequire(
|
|
61
|
+
path.join(configuration.puppeteerPath, "package.json")
|
|
62
|
+
);
|
|
63
|
+
const resolved = require.resolve("puppeteer");
|
|
64
|
+
const puppeteer = await import(pathToFileURL(resolved).href);
|
|
65
|
+
const launchArgs = ['--no-sandbox', '--disable-setuid-sandbox']
|
|
66
|
+
.concat(configuration.userOptions.chromeOptions)
|
|
67
|
+
.filter((item, index, self) => self.indexOf(item) === index);
|
|
59
68
|
|
|
60
69
|
if (configuration.userOptions['browserWebsocketUrl'] !== "") {
|
|
61
70
|
return await puppeteer.connect( {
|
|
@@ -64,7 +73,8 @@ exports.launchPuppeteer = async function (configuration) {
|
|
|
64
73
|
} else {
|
|
65
74
|
return await puppeteer.launch({
|
|
66
75
|
args: launchArgs,
|
|
67
|
-
headless: configuration.userOptions.isHeadless
|
|
76
|
+
headless: configuration.userOptions.isHeadless,
|
|
77
|
+
slowMo: configuration.userOptions.slowMo
|
|
68
78
|
});
|
|
69
79
|
}
|
|
70
80
|
}
|
|
@@ -74,7 +84,7 @@ exports.launchPuppeteer = async function (configuration) {
|
|
|
74
84
|
* @param {Object} page - The Puppeteer page object to configure.
|
|
75
85
|
* @param {UserOptions} userOptions - The user options to use.
|
|
76
86
|
*/
|
|
77
|
-
|
|
87
|
+
export async function configure(page, userOptions) {
|
|
78
88
|
if (userOptions.userAgent !== "") {
|
|
79
89
|
await page.setUserAgent(userOptions.userAgent)
|
|
80
90
|
}
|
|
@@ -93,16 +103,16 @@ exports.configure = async function (page, userOptions) {
|
|
|
93
103
|
* @param {Object} page - The Puppeteer page object to use for navigation.
|
|
94
104
|
* @param {Configuration} configuration - The configuration to use.
|
|
95
105
|
*/
|
|
96
|
-
|
|
106
|
+
export async function navigate(page, configuration) {
|
|
97
107
|
const navigationWaitForSelector = configuration.userOptions.navigationWaitForSelector;
|
|
98
108
|
const navigationWaitForXPath = configuration.userOptions.navigationWaitForXPath;
|
|
99
109
|
|
|
100
|
-
await page.goto(configuration.webPageUrl,
|
|
110
|
+
await page.goto(configuration.webPageUrl, getNavigationParameters(configuration));
|
|
101
111
|
|
|
102
112
|
if (navigationWaitForSelector !== "") {
|
|
103
|
-
await page.waitForSelector(navigationWaitForSelector,
|
|
113
|
+
await page.waitForSelector(navigationWaitForSelector, getWaitingParameters(configuration));
|
|
104
114
|
} else if (navigationWaitForXPath !== "") {
|
|
105
|
-
await page.waitForXPath(navigationWaitForXPath,
|
|
115
|
+
await page.waitForXPath(navigationWaitForXPath, getWaitingParameters(configuration));
|
|
106
116
|
} else {
|
|
107
117
|
await new Promise(r => setTimeout(r, 250));
|
|
108
118
|
}
|
|
@@ -114,7 +124,7 @@ exports.navigate = async function (page, configuration) {
|
|
|
114
124
|
* @param {UserOptions} configuration - The configuration to use.
|
|
115
125
|
* @returns {Object} - pdfOptions
|
|
116
126
|
*/
|
|
117
|
-
|
|
127
|
+
export async function getConfiguredPdfOptions(page, configuration) {
|
|
118
128
|
const pdfOptions = configuration.pdfOptions
|
|
119
129
|
|
|
120
130
|
if (pdfOptions['headerTemplateFile'] !== '') {
|
|
@@ -153,10 +163,10 @@ exports.getConfiguredPdfOptions = async function (page, configuration) {
|
|
|
153
163
|
* @returns {NavigationParameters}
|
|
154
164
|
* The extracted navigation parameters.
|
|
155
165
|
*/
|
|
156
|
-
|
|
166
|
+
export function getNavigationParameters(configuration) {
|
|
157
167
|
return {
|
|
158
168
|
timeout: configuration.userOptions.navigationTimeout,
|
|
159
|
-
|
|
169
|
+
waitUntil: configuration.userOptions.navigationWaitUntil
|
|
160
170
|
}
|
|
161
171
|
}
|
|
162
172
|
|
|
@@ -167,7 +177,7 @@ exports.getNavigationParameters = function (configuration) {
|
|
|
167
177
|
* @returns {WaitingParameters}
|
|
168
178
|
* The extracted waiting parameters.
|
|
169
179
|
*/
|
|
170
|
-
|
|
180
|
+
export function getWaitingParameters(configuration) {
|
|
171
181
|
return {
|
|
172
182
|
timeout: configuration.userOptions.navigationTimeout
|
|
173
183
|
}
|
data/lib/js/html-scraper.js
CHANGED
|
@@ -1,17 +1,17 @@
|
|
|
1
1
|
'use strict';
|
|
2
|
-
|
|
3
|
-
|
|
2
|
+
import {getConfiguration, launchPuppeteer, configure, navigate} from './dhalang.js';
|
|
3
|
+
import fs from 'node:fs';
|
|
4
4
|
|
|
5
5
|
const scrapeHtml = async () => {
|
|
6
|
-
const configuration =
|
|
6
|
+
const configuration = getConfiguration();
|
|
7
7
|
|
|
8
8
|
let browser;
|
|
9
9
|
let page;
|
|
10
10
|
try {
|
|
11
|
-
browser = await
|
|
11
|
+
browser = await launchPuppeteer(configuration);
|
|
12
12
|
page = await browser.newPage();
|
|
13
|
-
await
|
|
14
|
-
await
|
|
13
|
+
await configure(page, configuration.userOptions);
|
|
14
|
+
await navigate(page, configuration);
|
|
15
15
|
const html = await page.content();
|
|
16
16
|
fs.writeFileSync(configuration.tempFilePath, html);
|
|
17
17
|
} catch (error) {
|
data/lib/js/pdf-generator.js
CHANGED
|
@@ -1,17 +1,17 @@
|
|
|
1
1
|
'use strict';
|
|
2
|
-
|
|
2
|
+
import {getConfiguration, getConfiguredPdfOptions, launchPuppeteer, configure, navigate} from './dhalang.js';
|
|
3
3
|
|
|
4
4
|
const createPdf = async () => {
|
|
5
|
-
const configuration =
|
|
5
|
+
const configuration = getConfiguration();
|
|
6
6
|
|
|
7
7
|
let browser;
|
|
8
8
|
let page;
|
|
9
9
|
try {
|
|
10
|
-
browser = await
|
|
10
|
+
browser = await launchPuppeteer(configuration);
|
|
11
11
|
page = await browser.newPage();
|
|
12
|
-
await
|
|
13
|
-
await
|
|
14
|
-
const pdfOptions = await
|
|
12
|
+
await configure(page, configuration.userOptions);
|
|
13
|
+
await navigate(page, configuration);
|
|
14
|
+
const pdfOptions = await getConfiguredPdfOptions(page, configuration);
|
|
15
15
|
await page.pdf({
|
|
16
16
|
...{
|
|
17
17
|
path: configuration.tempFilePath
|
|
@@ -1,16 +1,16 @@
|
|
|
1
1
|
'use strict';
|
|
2
|
-
|
|
2
|
+
import {getConfiguration, launchPuppeteer, configure, navigate} from './dhalang.js';
|
|
3
3
|
|
|
4
4
|
const createScreenshot = async () => {
|
|
5
|
-
const configuration =
|
|
5
|
+
const configuration = getConfiguration();
|
|
6
6
|
|
|
7
7
|
let browser;
|
|
8
8
|
let page;
|
|
9
9
|
try {
|
|
10
|
-
browser = await
|
|
10
|
+
browser = await launchPuppeteer(configuration);
|
|
11
11
|
page = await browser.newPage();
|
|
12
|
-
await
|
|
13
|
-
await
|
|
12
|
+
await configure(page, configuration.userOptions);
|
|
13
|
+
await navigate(page, configuration);
|
|
14
14
|
|
|
15
15
|
await page.screenshot({
|
|
16
16
|
...{
|