@data-visuals/create 7.7.0 → 7.7.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json
CHANGED
|
@@ -1,338 +1,12 @@
|
|
|
1
|
-
// native
|
|
2
|
-
const path = require('path');
|
|
3
|
-
|
|
4
|
-
// packages
|
|
5
|
-
const colors = require('ansi-colors');
|
|
6
|
-
const fs = require('fs-extra');
|
|
7
|
-
const glob = require('fast-glob');
|
|
8
|
-
const puppeteer = require('puppeteer');
|
|
9
|
-
|
|
10
1
|
// internal
|
|
11
|
-
const
|
|
12
|
-
const config = require('../../project.config');
|
|
13
|
-
const { ensureSlash, logErrorMessage, logMessage } = require('../utils');
|
|
14
|
-
|
|
15
|
-
// used for screenshots
|
|
16
|
-
const TABLET_BREAKPOINT = process.env.TABLET_BREAKPOINT || 768;
|
|
17
|
-
const MOBILE_BREAKPOINT = process.env.MOBILE_BREAKPOINT || 475;
|
|
18
|
-
const viewportOpts = size => {
|
|
19
|
-
return {
|
|
20
|
-
width: Number(size === 'tablet' ? TABLET_BREAKPOINT : MOBILE_BREAKPOINT),
|
|
21
|
-
height: 1000,
|
|
22
|
-
deviceScaleFactor: 2,
|
|
23
|
-
};
|
|
24
|
-
};
|
|
25
|
-
|
|
26
|
-
// path where chrome is typically installed on MacOS
|
|
27
|
-
const CHROME_INSTALL_PATH =
|
|
28
|
-
'/Applications/Google Chrome.app/Contents/MacOS/Google Chrome';
|
|
29
|
-
|
|
30
|
-
const TAGS_PLACEHOLDER = ['subject-politics'];
|
|
31
|
-
|
|
32
|
-
const captureScreenshotOfElement = async (element, imagePath) => {
|
|
33
|
-
await fs.ensureDir(path.dirname(imagePath));
|
|
34
|
-
try {
|
|
35
|
-
await element.screenshot({ path: imagePath });
|
|
36
|
-
return path.basename(imagePath);
|
|
37
|
-
} catch (error) {
|
|
38
|
-
console.log(`Could not take screenshot of element ${imagePath}:`, error);
|
|
39
|
-
}
|
|
40
|
-
};
|
|
41
|
-
|
|
42
|
-
const createPreviews = async (params = { page: {}, outputPath: '' }) => {
|
|
43
|
-
const { page, outputPath } = params;
|
|
44
|
-
const body = await page.$('body');
|
|
45
|
-
if (body) {
|
|
46
|
-
// prevent edge-cropped screenshots
|
|
47
|
-
await page.addStyleTag({
|
|
48
|
-
content: 'body { padding: 10px; }',
|
|
49
|
-
});
|
|
50
|
-
// also hide CTAs
|
|
51
|
-
await page.addStyleTag({
|
|
52
|
-
content: '.button, .c-button { display: none; }',
|
|
53
|
-
});
|
|
54
|
-
await page.setViewport(viewportOpts('mobile'));
|
|
55
|
-
const small = await captureScreenshotOfElement(
|
|
56
|
-
body,
|
|
57
|
-
`${outputPath}preview-small.png`
|
|
58
|
-
);
|
|
59
|
-
await page.setViewport(viewportOpts('tablet'));
|
|
60
|
-
const large = await captureScreenshotOfElement(
|
|
61
|
-
body,
|
|
62
|
-
`${outputPath}preview-large.png`
|
|
63
|
-
);
|
|
64
|
-
return { small, large };
|
|
65
|
-
} else {
|
|
66
|
-
throw new Error(`Not found.`);
|
|
67
|
-
}
|
|
68
|
-
};
|
|
69
|
-
|
|
70
|
-
const getText = async (params = { key: '', page: {} }) => {
|
|
71
|
-
const { key, page } = params;
|
|
72
|
-
let value = '';
|
|
73
|
-
try {
|
|
74
|
-
value = await page.$eval(
|
|
75
|
-
`[data-${key}], meta[name="tt-graphic-${key}"]`,
|
|
76
|
-
el => {
|
|
77
|
-
if (el.hasAttribute('content')) {
|
|
78
|
-
return el.getAttribute('content');
|
|
79
|
-
} else {
|
|
80
|
-
return el.textContent;
|
|
81
|
-
}
|
|
82
|
-
}
|
|
83
|
-
);
|
|
84
|
-
} catch {
|
|
85
|
-
value = '';
|
|
86
|
-
}
|
|
87
|
-
|
|
88
|
-
return value;
|
|
89
|
-
};
|
|
90
|
-
|
|
91
|
-
const printWarnings = graphics => {
|
|
92
|
-
const { tags } = config;
|
|
93
|
-
const requiredKeys = ['alt-text', 'credits', 'source'];
|
|
94
|
-
|
|
95
|
-
// default tags used
|
|
96
|
-
if (JSON.stringify(tags) === JSON.stringify(TAGS_PLACEHOLDER)) {
|
|
97
|
-
logMessage(`Default tags used in project.config.js`);
|
|
98
|
-
}
|
|
99
|
-
|
|
100
|
-
for (const graphic of graphics) {
|
|
101
|
-
const { label } = graphic;
|
|
102
|
-
// loop through keys and warn accordingly
|
|
103
|
-
for (const [key, value] of Object.entries(graphic)) {
|
|
104
|
-
// missing values
|
|
105
|
-
if (requiredKeys.includes(key) && value.length === 0) {
|
|
106
|
-
logMessage(`Empty ${key} in ${label}`, 'red');
|
|
107
|
-
}
|
|
108
|
-
}
|
|
109
|
-
}
|
|
110
|
-
};
|
|
111
|
-
|
|
112
|
-
const parseGraphic = async (
|
|
113
|
-
params = { browser: {}, filepath: '', localURL: '' }
|
|
114
|
-
) => {
|
|
115
|
-
const { browser, filepath, localURL } = params;
|
|
116
|
-
|
|
117
|
-
// project config info
|
|
118
|
-
const {
|
|
119
|
-
bucket,
|
|
120
|
-
createMonth,
|
|
121
|
-
createYear,
|
|
122
|
-
lastBuildTime,
|
|
123
|
-
folder,
|
|
124
|
-
id,
|
|
125
|
-
parserOptions,
|
|
126
|
-
} = config;
|
|
127
|
-
|
|
128
|
-
const name = path.basename(filepath, path.extname(filepath));
|
|
129
|
-
const relativeDir = path.relative('./.tmp', filepath);
|
|
130
|
-
const parentDir = path.parse(relativeDir).dir;
|
|
131
|
-
const outputPath = path.join(paths.appDist, parentDir, '/');
|
|
132
|
-
|
|
133
|
-
// begin puppeteer navigation
|
|
134
|
-
const page = await browser.newPage();
|
|
135
|
-
const url =
|
|
136
|
-
parentDir === '.tmp' && name === 'index'
|
|
137
|
-
? `${localURL}`
|
|
138
|
-
: `${localURL}/${parentDir}/${name}.html`;
|
|
139
|
-
await page.goto(url, { waitUntil: 'load' });
|
|
140
|
-
const label = path.join(parentDir, `${name}.html`);
|
|
141
|
-
const projectPath = path.join(folder, parentDir);
|
|
142
|
-
const projectURL = ensureSlash(`https://${bucket}/${folder}/${parentDir}`);
|
|
143
|
-
|
|
144
|
-
// only consider HTML with data-graphic or data-feature attribute present
|
|
145
|
-
try {
|
|
146
|
-
await page.waitForSelector('[data-graphic], [data-feature]', {
|
|
147
|
-
timeout: 5000,
|
|
148
|
-
});
|
|
149
|
-
} catch (e) {
|
|
150
|
-
return false;
|
|
151
|
-
}
|
|
152
|
-
|
|
153
|
-
// get metadata
|
|
154
|
-
const title = await getText({ key: 'title', page });
|
|
155
|
-
const tags = await (await getText({ key: 'tags', page })).split(',');
|
|
156
|
-
let credits = await getText({ key: 'credit', page });
|
|
157
|
-
|
|
158
|
-
// determine type of metadata
|
|
159
|
-
// assume the type is graphic unless data-feature is present
|
|
160
|
-
const type = (await page.$('[data-feature]')) ? 'feature' : 'graphic';
|
|
161
|
-
|
|
162
|
-
// ignore projects with no title
|
|
163
|
-
if (title.length === 0) {
|
|
164
|
-
logMessage(
|
|
165
|
-
`${label} was skipped because no graphic or feature was title set.`
|
|
166
|
-
);
|
|
167
|
-
return false;
|
|
168
|
-
}
|
|
169
|
-
|
|
170
|
-
// create array from credits
|
|
171
|
-
if (credits.length > 0) {
|
|
172
|
-
// separate by commas or and
|
|
173
|
-
credits = credits.split(/, *| and */g);
|
|
174
|
-
} else {
|
|
175
|
-
credits = [];
|
|
176
|
-
}
|
|
177
|
-
|
|
178
|
-
// find all links
|
|
179
|
-
if (type == 'graphic') {
|
|
180
|
-
const caption = await getText({ key: 'caption', page });
|
|
181
|
-
const altText = await getText({ key: 'alt-text', page });
|
|
182
|
-
const note = await getText({ key: 'note', page });
|
|
183
|
-
let source = await getText({ key: 'source', page });
|
|
184
|
-
|
|
185
|
-
// commenting this out as a hot fix in December 2022 for issue #158
|
|
186
|
-
// // create array from source
|
|
187
|
-
// if (source.length > 0) {
|
|
188
|
-
// // separate by commas or and
|
|
189
|
-
// source = source.split(/, *| and */g);
|
|
190
|
-
// } else {
|
|
191
|
-
// source = [];
|
|
192
|
-
// }
|
|
193
|
-
|
|
194
|
-
const links = await page.$$eval('a', links =>
|
|
195
|
-
links.map(link => {
|
|
196
|
-
return {
|
|
197
|
-
url: link.getAttribute('href'),
|
|
198
|
-
text: link.textContent,
|
|
199
|
-
isCTA:
|
|
200
|
-
link.classList.contains('button') ||
|
|
201
|
-
link.classList.contains('.c-button'),
|
|
202
|
-
};
|
|
203
|
-
})
|
|
204
|
-
);
|
|
205
|
-
|
|
206
|
-
// take screenshots of page
|
|
207
|
-
const { small, large } = await createPreviews({
|
|
208
|
-
page,
|
|
209
|
-
outputPath,
|
|
210
|
-
});
|
|
211
|
-
|
|
212
|
-
// check if appleNewsIgnore is specified
|
|
213
|
-
let showInAppleNews = true;
|
|
214
|
-
if (parserOptions) {
|
|
215
|
-
const { appleNewsIgnore } = parserOptions;
|
|
216
|
-
showInAppleNews =
|
|
217
|
-
!appleNewsIgnore.includes(parentDir) &&
|
|
218
|
-
!appleNewsIgnore.includes(label);
|
|
219
|
-
}
|
|
220
|
-
|
|
221
|
-
// all graphic data
|
|
222
|
-
return {
|
|
223
|
-
type,
|
|
224
|
-
title,
|
|
225
|
-
altText,
|
|
226
|
-
bucket,
|
|
227
|
-
projectPath,
|
|
228
|
-
projectURL,
|
|
229
|
-
caption,
|
|
230
|
-
createMonth,
|
|
231
|
-
createYear,
|
|
232
|
-
credits,
|
|
233
|
-
folder,
|
|
234
|
-
id,
|
|
235
|
-
lastBuildTime,
|
|
236
|
-
label,
|
|
237
|
-
links,
|
|
238
|
-
note,
|
|
239
|
-
previews: {
|
|
240
|
-
large: projectURL + large,
|
|
241
|
-
small: projectURL + small,
|
|
242
|
-
},
|
|
243
|
-
showInAppleNews,
|
|
244
|
-
source,
|
|
245
|
-
tags,
|
|
246
|
-
};
|
|
247
|
-
}
|
|
248
|
-
|
|
249
|
-
if (type == 'feature') {
|
|
250
|
-
// all feature data
|
|
251
|
-
return {
|
|
252
|
-
type,
|
|
253
|
-
title,
|
|
254
|
-
bucket,
|
|
255
|
-
projectPath,
|
|
256
|
-
projectURL,
|
|
257
|
-
createMonth,
|
|
258
|
-
createYear,
|
|
259
|
-
credits,
|
|
260
|
-
folder,
|
|
261
|
-
id,
|
|
262
|
-
lastBuildTime,
|
|
263
|
-
label,
|
|
264
|
-
tags,
|
|
265
|
-
};
|
|
266
|
-
}
|
|
267
|
-
};
|
|
2
|
+
const { logMessage } = require('../utils');
|
|
268
3
|
|
|
269
4
|
module.exports = async localURL => {
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
const pages = await glob('**/*.html', {
|
|
274
|
-
absolute: true,
|
|
275
|
-
cwd: './.tmp',
|
|
276
|
-
recursive: true,
|
|
277
|
-
ignore: parserOptions.metadataIgnore,
|
|
278
|
-
});
|
|
279
|
-
|
|
280
|
-
// spin up headless browser using local chrome
|
|
281
|
-
const browser = await puppeteer
|
|
282
|
-
.launch({
|
|
283
|
-
executablePath: process.env.CHROME_INSTALL_PATH || CHROME_INSTALL_PATH,
|
|
284
|
-
// Hot fix on puppeteer in Feb 2025
|
|
285
|
-
headless: false,
|
|
286
|
-
})
|
|
287
|
-
.catch(err => {
|
|
288
|
-
logErrorMessage(err);
|
|
289
|
-
throw new Error(
|
|
290
|
-
colors.yellow(
|
|
291
|
-
`Could not find the chrome installed at '${CHROME_INSTALL_PATH}'.\nDo you have a local version of Chrome installed? \n If so, navigate to chrome://version/ in your Chrome browser and copy the path listed for Executable Path. Then rerun this process with your path:\n\nCHROME_INSTALL_PATH="local/path/to/chrome" npm run parse.\n`
|
|
292
|
-
)
|
|
293
|
-
);
|
|
294
|
-
});
|
|
295
|
-
logMessage(`Parsing graphics in a headless browser...`, 'cyan');
|
|
296
|
-
|
|
297
|
-
// parse each HTML page
|
|
298
|
-
const graphics = await Promise.all(
|
|
299
|
-
pages.map(filepath =>
|
|
300
|
-
parseGraphic({
|
|
301
|
-
browser,
|
|
302
|
-
filepath,
|
|
303
|
-
localURL,
|
|
304
|
-
config,
|
|
305
|
-
})
|
|
306
|
-
)
|
|
307
|
-
);
|
|
308
|
-
|
|
309
|
-
// filter skipped HTML files
|
|
310
|
-
const filtered = graphics.filter(
|
|
311
|
-
graphic => typeof graphic.title === 'string'
|
|
5
|
+
logMessage(
|
|
6
|
+
'Graphics metadata generation is currently disabled. Puppeteer has been removed pending a permanent solution for Apple News and Newspack graphics plugin.',
|
|
7
|
+
'yellow'
|
|
312
8
|
);
|
|
313
9
|
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
// output path
|
|
317
|
-
const manifest = `${paths.appDist}/manifest.json`;
|
|
318
|
-
|
|
319
|
-
// output JSON of all metadata
|
|
320
|
-
try {
|
|
321
|
-
await fs.outputJson(manifest, filtered);
|
|
322
|
-
} catch (err) {
|
|
323
|
-
throw new Error(err);
|
|
324
|
-
}
|
|
325
|
-
|
|
326
|
-
// print output info in terminal
|
|
327
|
-
if (filtered.length > 0) {
|
|
328
|
-
logMessage(`Generated metadata for ${filtered.length} item(s)`, 'green');
|
|
329
|
-
console.log(`✔ ${manifest}`);
|
|
330
|
-
} else {
|
|
331
|
-
logMessage(
|
|
332
|
-
`No metadata generated. Could not find the data-graphic or data-feature attribute in any HTML files.`,
|
|
333
|
-
'red'
|
|
334
|
-
);
|
|
335
|
-
}
|
|
336
|
-
|
|
337
|
-
await browser.close();
|
|
10
|
+
// Return empty array so parse.js doesn't fail
|
|
11
|
+
return [];
|
|
338
12
|
};
|
|
@@ -79,6 +79,12 @@ async function writeToSheet(
|
|
|
79
79
|
|
|
80
80
|
// update log of past data visuals works with project info
|
|
81
81
|
let updateLogSheet = async (mainPath, config) => {
|
|
82
|
+
// Check if manifest exists first
|
|
83
|
+
if (!fs.existsSync(`${paths.appDist}/manifest.json`)) {
|
|
84
|
+
console.log('No manifest.json found, skipping detailed metadata logging');
|
|
85
|
+
return;
|
|
86
|
+
}
|
|
87
|
+
|
|
82
88
|
// read manifest file, which has metadata about the project
|
|
83
89
|
const manifest = fs.readFileSync(`${paths.appDist}/manifest.json`, 'utf8');
|
|
84
90
|
|
|
@@ -2,18 +2,18 @@
|
|
|
2
2
|
<hr class="has-bg-gray-light has-giant-btm-marg" />
|
|
3
3
|
<h2 class="is-sr-only">Information about the authors</h2>
|
|
4
4
|
|
|
5
|
-
<div class="c-author-info__container l-flex l-flex-column
|
|
5
|
+
<div class="c-author-info__container l-flex l-flex-column">
|
|
6
6
|
<div
|
|
7
7
|
id="staff-author"
|
|
8
8
|
class="c-author-info__container l-flex l-flex-column"
|
|
9
9
|
>
|
|
10
10
|
{% for author in context['staff_authors'] %}
|
|
11
|
-
<div class="c-author-info__trib-author-container l-flex
|
|
11
|
+
<div class="c-author-info__trib-author-container l-flex">
|
|
12
12
|
{% if author['author_photo'] %}
|
|
13
13
|
<a class="t-links-unset" href="{{ author['author_link'] }}">
|
|
14
14
|
<noscript>
|
|
15
15
|
<img
|
|
16
|
-
width="
|
|
16
|
+
width="100"
|
|
17
17
|
height="100"
|
|
18
18
|
alt="{{ author['author_name'] }}’s staff photo"
|
|
19
19
|
src="{{ author['author_photo'] }}"
|
|
@@ -23,7 +23,7 @@
|
|
|
23
23
|
alt="{{ author['author_name'] }}’s staff photo"
|
|
24
24
|
class="l-display-block js-lazy-image has-bg-gray-dark js-lazy-image--loaded"
|
|
25
25
|
data-src="{{ author['author_photo'] }}"
|
|
26
|
-
width="
|
|
26
|
+
width="100"
|
|
27
27
|
height="100"
|
|
28
28
|
src="{{ author['author_photo'] }}"
|
|
29
29
|
/>
|
|
@@ -44,7 +44,7 @@
|
|
|
44
44
|
class="c-author-info__contacts-container l-flex l-flex-column t-size-xs t-sans"
|
|
45
45
|
>
|
|
46
46
|
{% if author['author_email'] %}
|
|
47
|
-
<p class="l-flex">
|
|
47
|
+
<p class="l-flex ">
|
|
48
48
|
<a
|
|
49
49
|
tabindex="-1"
|
|
50
50
|
href="mailto:{{ author['author_email'] }}"
|