@flourish/sdk 3.17.1 → 3.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md~ +473 -0
- package/RELEASE_NOTES.md +9 -0
- package/RELEASE_NOTES.md~ +372 -0
- package/lib/cmd/publish.js +4 -2
- package/lib/log.js +7 -7
- package/lib/sdk.js +34 -22
- package/lib/sdk.js~ +540 -0
- package/lib/validate_config.js +2 -1
- package/lib/validate_config.js~ +437 -0
- package/package-lock.json~ +2669 -0
- package/package.json +4 -4
- package/package.json~ +53 -0
- package/server/data.js +40 -5
- package/server/index.js~ +540 -0
- package/site/embedded.js +1 -1
- package/site/images/flourish_in_canva.png +0 -0
- package/site/script.js +2 -2
- package/site/sdk.css +1 -1
- package/test/lib/validate_config.js +11 -2
- package/test/lib/validate_config.js~ +1006 -0
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@flourish/sdk",
|
|
3
|
-
"version": "3.
|
|
3
|
+
"version": "3.18.0",
|
|
4
4
|
"description": "The Flourish SDK",
|
|
5
5
|
"module": "src/index.js",
|
|
6
6
|
"scripts": {
|
|
@@ -23,7 +23,6 @@
|
|
|
23
23
|
"@rollup/plugin-commonjs": "^17.1.0",
|
|
24
24
|
"archiver": "^5.0.2",
|
|
25
25
|
"chokidar": "^3.4.3",
|
|
26
|
-
"colors": "^1.4.0",
|
|
27
26
|
"cross-spawn": "^7.0.3",
|
|
28
27
|
"d3-dsv": "^2.0.0",
|
|
29
28
|
"express": "^4.17.1",
|
|
@@ -32,8 +31,9 @@
|
|
|
32
31
|
"js-yaml": "^3.14.0",
|
|
33
32
|
"minimist": "^1.2.5",
|
|
34
33
|
"ncp": "^2.0.0",
|
|
35
|
-
"node-fetch": "^2.6.
|
|
34
|
+
"node-fetch": "^2.6.7",
|
|
36
35
|
"parse5": "^6.0.1",
|
|
36
|
+
"picocolors": "^1.0.0",
|
|
37
37
|
"read": "^1.0.7",
|
|
38
38
|
"resolve": "^1.18.1",
|
|
39
39
|
"rewrite-links": "^1.1.0",
|
|
@@ -45,7 +45,7 @@
|
|
|
45
45
|
"devDependencies": {
|
|
46
46
|
"@rollup/plugin-node-resolve": "^9.0.0",
|
|
47
47
|
"d3-request": "^1.0.6",
|
|
48
|
-
"mocha": "^9.
|
|
48
|
+
"mocha": "^9.2.0",
|
|
49
49
|
"npm-audit-resolver": "^2.3.0",
|
|
50
50
|
"rollup": "^2.32.1",
|
|
51
51
|
"sinon": "^9.2.0",
|
package/package.json~
ADDED
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@flourish/sdk",
|
|
3
|
+
<<<<<<< HEAD
|
|
4
|
+
"version": "3.13.1",
|
|
5
|
+
=======
|
|
6
|
+
"version": "3.14.0",
|
|
7
|
+
>>>>>>> sdk-watch-command
|
|
8
|
+
"description": "The Flourish SDK",
|
|
9
|
+
"module": "src/index.js",
|
|
10
|
+
"scripts": {
|
|
11
|
+
"prepare": "cd .. && make sdk_clean sdk",
|
|
12
|
+
"test": "mocha"
|
|
13
|
+
},
|
|
14
|
+
"bin": {
|
|
15
|
+
"flourish": "bin/flourish"
|
|
16
|
+
},
|
|
17
|
+
"author": "Kiln Enterprises Ltd",
|
|
18
|
+
"license": "SEE LICENSE IN LICENSE.md",
|
|
19
|
+
"repository": "kiln/flourish-sdk",
|
|
20
|
+
"dependencies": {
|
|
21
|
+
"@flourish/semver": "^1.0.1",
|
|
22
|
+
"archiver": "^3.0.0",
|
|
23
|
+
"chokidar": "^3.0.1",
|
|
24
|
+
"colors": "^1.3.3",
|
|
25
|
+
"cross-spawn": "^6.0.5",
|
|
26
|
+
"d3-dsv": "^1.1.1",
|
|
27
|
+
"express": "^4.17.1",
|
|
28
|
+
"handlebars": "^4.1.2",
|
|
29
|
+
"js-yaml": "^3.13.1",
|
|
30
|
+
"minimist": "^1.2.0",
|
|
31
|
+
"ncp": "^2.0.0",
|
|
32
|
+
"parse5": "^5.1.0",
|
|
33
|
+
"read": "^1.0.7",
|
|
34
|
+
"request": "^2.88.0",
|
|
35
|
+
"resolve": "^1.11.1",
|
|
36
|
+
"rewrite-links": "^1.1.0",
|
|
37
|
+
"shell-quote": "^1.6.1",
|
|
38
|
+
"tmp": "^0.1.0",
|
|
39
|
+
"ws": "^7.0.1"
|
|
40
|
+
},
|
|
41
|
+
"devDependencies": {
|
|
42
|
+
"d3-request": "^1.0.6",
|
|
43
|
+
"mocha": "^6.1.4",
|
|
44
|
+
"rollup": "^1.16.2",
|
|
45
|
+
"rollup-plugin-node-resolve": "^5.1.0",
|
|
46
|
+
"rollup-plugin-uglify": "^6.0.2",
|
|
47
|
+
"sinon": "^8.1.1",
|
|
48
|
+
"tempy": "^0.3.0"
|
|
49
|
+
},
|
|
50
|
+
"engines": {
|
|
51
|
+
"node": ">=8.3"
|
|
52
|
+
}
|
|
53
|
+
}
|
package/server/data.js
CHANGED
|
@@ -175,12 +175,43 @@ function extractData(data_binding, data_by_id, column_types_by_id, template_data
|
|
|
175
175
|
|
|
176
176
|
function getColumnTypesForData(data) {
|
|
177
177
|
return transposeNestedArray(data)
|
|
178
|
-
.map(function(
|
|
179
|
-
const
|
|
178
|
+
.map(function(column, i) {
|
|
179
|
+
const sliced_column = getSlicedData(column);
|
|
180
|
+
const sample_size = 1000;
|
|
181
|
+
let sample_data;
|
|
182
|
+
if (sliced_column.length > (sample_size * 2)) sample_data = getRandomSeededSample(sliced_column, sample_size);
|
|
183
|
+
else sample_data = sliced_column;
|
|
184
|
+
const type_id = interpretColumn(sample_data)[0].id;
|
|
180
185
|
return { type_id: type_id, index: i, output_format_id: type_id };
|
|
181
186
|
});
|
|
182
187
|
}
|
|
183
188
|
|
|
189
|
+
// Returns a random seeded sample of column values based on the column length.
|
|
190
|
+
// The sample is consistent and will update if the length of column changes.
|
|
191
|
+
function getRandomSeededSample(column, sample_size) {
|
|
192
|
+
if (column.length <= sample_size * 2) return column;
|
|
193
|
+
const rng = mulberry32(column.length);
|
|
194
|
+
|
|
195
|
+
while (column.length > sample_size) {
|
|
196
|
+
const random_index = Math.floor(rng() * column.length);
|
|
197
|
+
|
|
198
|
+
column.splice(random_index, 1);
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
return column;
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
// Seeded RNG implementation taken from https://github.com/bryc/code/blob/master/jshash/PRNGs.md#mulberry32
|
|
205
|
+
function mulberry32(seed) {
|
|
206
|
+
let a = seed;
|
|
207
|
+
return function() {
|
|
208
|
+
a |= 0; a = a + 0x6D2B79F5 | 0;
|
|
209
|
+
var t = Math.imul(a ^ a >>> 15, 1 | a);
|
|
210
|
+
t = t + Math.imul(t ^ t >>> 7, 61 | t) ^ t;
|
|
211
|
+
return ((t ^ t >>> 14) >>> 0) / 4294967296;
|
|
212
|
+
};
|
|
213
|
+
}
|
|
214
|
+
|
|
184
215
|
function trimTrailingEmptyRows(data) {
|
|
185
216
|
for (var i = data.length; i-- > 1;) {
|
|
186
217
|
if (!data[i] || !data[i].length || (Array.isArray(data[i]) && data[i].findIndex(function(col) { return col !== null && col !== ""; }) == -1)) {
|
|
@@ -194,9 +225,11 @@ function trimTrailingEmptyRows(data) {
|
|
|
194
225
|
function dropReturnCharacters(data) {
|
|
195
226
|
for (const row of data) {
|
|
196
227
|
for (let i = 0; i < row.length; i++) {
|
|
197
|
-
//
|
|
198
|
-
//
|
|
199
|
-
|
|
228
|
+
// Due to a bug in HoT, pasting long lines from Excel can lead to the addition of
|
|
229
|
+
// a newline character and a space *before* a space character.
|
|
230
|
+
// This leads to a pattern of new line character followed by two spaces.
|
|
231
|
+
// Here we identify that pattern and revert it.
|
|
232
|
+
row[i] = row[i].replace(/(\r\n|\n|\r) {2}/g, " ");
|
|
200
233
|
}
|
|
201
234
|
}
|
|
202
235
|
return data;
|
|
@@ -262,8 +295,10 @@ function interpretColumn(arr) {
|
|
|
262
295
|
exports.dropReturnCharacters = dropReturnCharacters;
|
|
263
296
|
exports.extractData = extractData;
|
|
264
297
|
exports.getColumnTypesForData = getColumnTypesForData;
|
|
298
|
+
exports.getRandomSeededSample = getRandomSeededSample;
|
|
265
299
|
exports.getSlicedData = getSlicedData;
|
|
266
300
|
exports.interpretColumn = interpretColumn;
|
|
301
|
+
exports.mulberry32 = mulberry32;
|
|
267
302
|
exports.transposeNestedArray = transposeNestedArray;
|
|
268
303
|
exports.trimTrailingEmptyRows = trimTrailingEmptyRows;
|
|
269
304
|
exports.trimWhitespace = trimWhitespace;
|
package/server/index.js~
ADDED
|
@@ -0,0 +1,540 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
|
|
3
|
+
// Modules
|
|
4
|
+
const crypto = require("crypto"),
|
|
5
|
+
fs = require("fs"),
|
|
6
|
+
path = require("path"),
|
|
7
|
+
|
|
8
|
+
cross_spawn = require("cross-spawn"),
|
|
9
|
+
chokidar = require("chokidar"),
|
|
10
|
+
d3_dsv = require("d3-dsv"),
|
|
11
|
+
express = require("express"),
|
|
12
|
+
handlebars = require("handlebars"),
|
|
13
|
+
shell_quote = require("shell-quote"),
|
|
14
|
+
ws = require("ws"),
|
|
15
|
+
yaml = require("js-yaml"),
|
|
16
|
+
|
|
17
|
+
columns = require("./columns"),
|
|
18
|
+
comms_js = require("./comms_js"),
|
|
19
|
+
data_utils = require("./data"),
|
|
20
|
+
index_html = require("./index_html"),
|
|
21
|
+
json = require("./json"),
|
|
22
|
+
|
|
23
|
+
log = require("../lib/log"),
|
|
24
|
+
sdk = require("../lib/sdk");
|
|
25
|
+
|
|
26
|
+
const TA = require("parse5/lib/tree-adapters/default.js");
|
|
27
|
+
|
|
28
|
+
// Generate a static prefix randomly
|
|
29
|
+
//
|
|
30
|
+
// Use a different prefix for /preview, to catch the situation where the template
|
|
31
|
+
// developer mistakenly prepends a / to the static prefix.
|
|
32
|
+
const static_prefix = crypto.randomBytes(15).toString("base64").replace(/[+/]/g, (c) => ({ "/": "_", "+": "-" })[c]),
|
|
33
|
+
preview_static_prefix = crypto.randomBytes(15).toString("base64").replace(/[+/]/g, (c) => ({ "/": "_", "+": "-" })[c]);
|
|
34
|
+
|
|
35
|
+
function loadFile(path_parts, options) {
|
|
36
|
+
return new Promise(function(resolve, reject) {
|
|
37
|
+
const file_path = path.join(...path_parts),
|
|
38
|
+
filename = path_parts[path_parts.length - 1];
|
|
39
|
+
|
|
40
|
+
function fail(message, error) {
|
|
41
|
+
if ("default" in options) {
|
|
42
|
+
log.warn(message, `Proceeding without ${filename}...`);
|
|
43
|
+
if (typeof options.default === "function") {
|
|
44
|
+
return resolve(options.default());
|
|
45
|
+
}
|
|
46
|
+
return resolve(options.default);
|
|
47
|
+
}
|
|
48
|
+
log.problem(message, error.message);
|
|
49
|
+
reject(error);
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
function succeed(result) {
|
|
53
|
+
if (!options.silentSuccess) log.success(`Loaded ${filename}`);
|
|
54
|
+
resolve(result);
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
fs.readFile(file_path, "utf8", function(error, loaded_text) {
|
|
58
|
+
if (error) return fail(`Failed to load ${file_path}`, error);
|
|
59
|
+
switch (options.type) {
|
|
60
|
+
case "json":
|
|
61
|
+
try { return succeed(JSON.parse(loaded_text)); }
|
|
62
|
+
catch (error) {
|
|
63
|
+
return fail(`Uh-oh! There's a problem with your ${filename} file.`, error);
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
case "yaml":
|
|
67
|
+
try { return succeed(yaml.safeLoad(loaded_text)); }
|
|
68
|
+
catch (error) {
|
|
69
|
+
return fail(`Uh-oh! There's a problem with your ${filename} file.`, error);
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
default:
|
|
73
|
+
return succeed(loaded_text);
|
|
74
|
+
}
|
|
75
|
+
});
|
|
76
|
+
});
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
function loadSDKTemplate() {
|
|
80
|
+
return loadFile([__dirname, "views", "index.html"], { silentSuccess: true })
|
|
81
|
+
.then((template_text) => handlebars.compile(template_text));
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
function loadTemplateText(template_dir) {
|
|
85
|
+
return loadFile([template_dir, "index.html"], {
|
|
86
|
+
default: () => loadFile([__dirname, "views", "default_template_index.html"], {
|
|
87
|
+
silentSuccess: true
|
|
88
|
+
})
|
|
89
|
+
});
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
function loadJavaScript(template_dir) {
|
|
93
|
+
return loadFile([template_dir, "template.js"], {});
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
function loadSettings(template_dir) {
|
|
97
|
+
return sdk.readAndValidateConfig(template_dir);
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
function listDataTables(template_dir) {
|
|
101
|
+
return new Promise(function(resolve, reject) {
|
|
102
|
+
fs.readdir(path.join(template_dir, "data"), function(error, filenames) {
|
|
103
|
+
if (error) {
|
|
104
|
+
if (error.code === "ENOENT") return resolve([]);
|
|
105
|
+
return reject(error);
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
const data_files = [];
|
|
109
|
+
for (let filename of filenames) {
|
|
110
|
+
if (!filename.endsWith(".csv")) continue;
|
|
111
|
+
|
|
112
|
+
var name = filename.substr(0, filename.length - 4);
|
|
113
|
+
data_files.push(name);
|
|
114
|
+
}
|
|
115
|
+
resolve(data_files);
|
|
116
|
+
});
|
|
117
|
+
});
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
function getData(template_dir, data_tables) {
|
|
121
|
+
return Promise.all(data_tables.map((data_table) => getDataTable(template_dir, data_table)))
|
|
122
|
+
.then((data_array) => {
|
|
123
|
+
const data_by_name = {};
|
|
124
|
+
const column_types_by_name = {};
|
|
125
|
+
for (var i = 0; i < data_tables.length; i++) {
|
|
126
|
+
data_by_name[data_tables[i]] = data_array[i];
|
|
127
|
+
}
|
|
128
|
+
for (const data_table in data_by_name) {
|
|
129
|
+
const data = data_by_name[data_table];
|
|
130
|
+
column_types_by_name[data_table] = data_utils.transposeNestedArray(data)
|
|
131
|
+
.map(function(d, i) { return { type_id: data_utils.interpretColumn(d)[0].id, index: i }; });
|
|
132
|
+
}
|
|
133
|
+
return { data: data_by_name, column_types_by_name };
|
|
134
|
+
});
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
function getDataTable(template_dir, data_table) {
|
|
138
|
+
return new Promise(function(resolve, reject) {
|
|
139
|
+
fs.readFile(path.join(template_dir, "data", data_table + ".csv"), "utf8", function(error, csv_text) {
|
|
140
|
+
if (error) return reject(error);
|
|
141
|
+
if (csv_text.charAt(0) === "\uFEFF") csv_text = csv_text.substr(1);
|
|
142
|
+
resolve(d3_dsv.csvParseRows(csv_text));
|
|
143
|
+
});
|
|
144
|
+
});
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
function parseDataBindings(data_bindings, data_tables) {
|
|
148
|
+
if (!data_bindings) return {1: {}};
|
|
149
|
+
|
|
150
|
+
// Use the names as ids
|
|
151
|
+
const name_by_id = {};
|
|
152
|
+
for (let name of data_tables) name_by_id[name] = name;
|
|
153
|
+
|
|
154
|
+
// Collect parsed bindings by dataset
|
|
155
|
+
const data_bindings_by_dataset = {};
|
|
156
|
+
for (let binding of data_bindings) {
|
|
157
|
+
let dataset = binding.dataset;
|
|
158
|
+
if (!dataset) continue;
|
|
159
|
+
|
|
160
|
+
if (!data_bindings_by_dataset[dataset]) data_bindings_by_dataset[dataset] = {};
|
|
161
|
+
data_bindings_by_dataset[dataset][binding.key] = columns.parseDataBinding(binding, name_by_id);
|
|
162
|
+
}
|
|
163
|
+
return { 1: data_bindings_by_dataset };
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
function documentFragment(elements) {
|
|
167
|
+
const fragment = TA.createDocumentFragment();
|
|
168
|
+
for (const element of elements) {
|
|
169
|
+
TA.appendChild(fragment, element);
|
|
170
|
+
}
|
|
171
|
+
return fragment;
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
function scriptElementInline(code) {
|
|
175
|
+
const element = TA.createElement("script", "http://www.w3.org/1999/xhtml", []);
|
|
176
|
+
TA.insertText(element, code);
|
|
177
|
+
return element;
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
function scriptElementExternal(url) {
|
|
181
|
+
return TA.createElement("script", "http://www.w3.org/1999/xhtml", [{ name: "src", value: url }]);
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
function loadTemplate(template_dir, sdk_template, build_failed) {
|
|
186
|
+
return Promise.all([
|
|
187
|
+
listDataTables(template_dir),
|
|
188
|
+
loadSettings(template_dir),
|
|
189
|
+
])
|
|
190
|
+
.then(([data_tables, settings]) => {
|
|
191
|
+
const data_bindings = parseDataBindings(settings.data, data_tables);
|
|
192
|
+
return Promise.all([
|
|
193
|
+
settings, data_bindings, data_tables,
|
|
194
|
+
previewInitJs(template_dir, data_bindings["1"], data_tables),
|
|
195
|
+
loadTemplateText(template_dir),
|
|
196
|
+
loadJavaScript(template_dir)
|
|
197
|
+
]);
|
|
198
|
+
})
|
|
199
|
+
.then(([
|
|
200
|
+
settings, data_bindings, data_tables,
|
|
201
|
+
preview_init_js, template_text, template_js
|
|
202
|
+
]) => {
|
|
203
|
+
const page_params = {
|
|
204
|
+
// Always use ID of 1 for SDK
|
|
205
|
+
visualisation: { id: 1, can_edit: true },
|
|
206
|
+
visualisation_js: "new Flourish.Visualisation('1', 0," + json.safeStringify({
|
|
207
|
+
data_bindings: data_bindings,
|
|
208
|
+
data_tables: data_tables,
|
|
209
|
+
}) + ")",
|
|
210
|
+
settings: json.safeStringify(settings.settings || []),
|
|
211
|
+
data_bindings: json.safeStringify(settings.data || []),
|
|
212
|
+
template_name: settings.name || "Untitled template",
|
|
213
|
+
template_version: settings.version,
|
|
214
|
+
template_author: settings.author || "",
|
|
215
|
+
build_failed: build_failed && build_failed.size > 0
|
|
216
|
+
};
|
|
217
|
+
|
|
218
|
+
const script = documentFragment([
|
|
219
|
+
scriptElementInline("window.Flourish = " + json.safeStringify({
|
|
220
|
+
static_prefix, environment: "sdk"
|
|
221
|
+
}) + ";"),
|
|
222
|
+
scriptElementExternal("/template.js"),
|
|
223
|
+
scriptElementExternal("/comms.js"),
|
|
224
|
+
scriptElementExternal("/embedded.js"),
|
|
225
|
+
]);
|
|
226
|
+
|
|
227
|
+
const preview_script = documentFragment([
|
|
228
|
+
scriptElementInline("window.Flourish = " + json.safeStringify({
|
|
229
|
+
static_prefix: preview_static_prefix, environment: "sdk"
|
|
230
|
+
}) + ";"),
|
|
231
|
+
scriptElementExternal("/template.js"),
|
|
232
|
+
scriptElementExternal("/comms.js"),
|
|
233
|
+
scriptElementExternal("/embedded.js"),
|
|
234
|
+
scriptElementExternal("/talk_to_server.js"),
|
|
235
|
+
scriptElementInline("_Flourish_talkToServer();"),
|
|
236
|
+
scriptElementInline(preview_init_js),
|
|
237
|
+
]);
|
|
238
|
+
|
|
239
|
+
return Promise.all([
|
|
240
|
+
sdk_template(page_params),
|
|
241
|
+
index_html.render(template_text, {
|
|
242
|
+
title: "Flourish SDK template preview blah",
|
|
243
|
+
static: static_prefix,
|
|
244
|
+
parsed_script: script
|
|
245
|
+
}),
|
|
246
|
+
index_html.render(template_text, {
|
|
247
|
+
title: "Flourish SDK template preview flerm",
|
|
248
|
+
static: preview_static_prefix,
|
|
249
|
+
parsed_script: preview_script,
|
|
250
|
+
}),
|
|
251
|
+
template_js,
|
|
252
|
+
sdk.buildRules(template_dir),
|
|
253
|
+
]);
|
|
254
|
+
})
|
|
255
|
+
.then(([sdk_rendered, template_rendered, preview_rendered, template_js, build_rules]) => ({
|
|
256
|
+
sdk_rendered, template_rendered, preview_rendered, template_js, build_rules
|
|
257
|
+
}));
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
function previewInitJs(template_dir, data_bindings, data_tables) {
|
|
261
|
+
return getData(template_dir, data_tables).then(({data, column_types_by_name}) => {
|
|
262
|
+
const prepared_data = {};
|
|
263
|
+
for (let dataset in data_bindings) {
|
|
264
|
+
prepared_data[dataset] = data_utils.extractData(data_bindings[dataset], data, column_types_by_name);
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
const column_names = {};
|
|
268
|
+
const type_information = {};
|
|
269
|
+
for (let dataset in prepared_data) {
|
|
270
|
+
column_names[dataset] = prepared_data[dataset].column_names;
|
|
271
|
+
type_information[dataset] = prepared_data[dataset].type_information;
|
|
272
|
+
}
|
|
273
|
+
|
|
274
|
+
return `
|
|
275
|
+
var _Flourish_data_column_names = ${json.safeStringify(column_names)},
|
|
276
|
+
_Flourish_data_type_information = ${json.safeStringify(type_information)},
|
|
277
|
+
_Flourish_data = ${json.javaScriptStringify(prepared_data)};
|
|
278
|
+
for (var _Flourish_dataset in _Flourish_data) {
|
|
279
|
+
window.template.data[_Flourish_dataset] = _Flourish_data[_Flourish_dataset];
|
|
280
|
+
window.template.data[_Flourish_dataset].column_names = _Flourish_data_column_names[_Flourish_dataset];
|
|
281
|
+
window.template.data[_Flourish_dataset].type_information = _Flourish_data_type_information[_Flourish_dataset];
|
|
282
|
+
}
|
|
283
|
+
window.template.draw();
|
|
284
|
+
`;
|
|
285
|
+
});
|
|
286
|
+
}
|
|
287
|
+
|
|
288
|
+
function tryToOpen(url) {
|
|
289
|
+
// If it’s available and works, use /usr/bin/open to open
|
|
290
|
+
// the URL. If not just prompt the user to open it.
|
|
291
|
+
try {
|
|
292
|
+
cross_spawn.spawn("/usr/bin/open", [url])
|
|
293
|
+
.on("exit", function(exit_code) {
|
|
294
|
+
if (exit_code != 0) {
|
|
295
|
+
log.success("Now open " + url + " in your web browser!");
|
|
296
|
+
}
|
|
297
|
+
else {
|
|
298
|
+
log.success("Opened browser window to " + url);
|
|
299
|
+
}
|
|
300
|
+
})
|
|
301
|
+
.on("error", function() {
|
|
302
|
+
log.success("Now open " + url + " in your web browser!");
|
|
303
|
+
});
|
|
304
|
+
}
|
|
305
|
+
catch (error) {
|
|
306
|
+
log.success("Now open " + url + " in your web browser!");
|
|
307
|
+
}
|
|
308
|
+
}
|
|
309
|
+
|
|
310
|
+
function isPrefix(a, b) {
|
|
311
|
+
if (a.length > b.length) return false;
|
|
312
|
+
for (let i = 0; i < a.length; i++) {
|
|
313
|
+
if (a[i] !== b[i]) return false;
|
|
314
|
+
}
|
|
315
|
+
return true;
|
|
316
|
+
}
|
|
317
|
+
|
|
318
|
+
function splitPath(p) {
|
|
319
|
+
return p.split(path.sep).filter(c => c != "");
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
|
|
323
|
+
module.exports = function(template_dir, options) {
|
|
324
|
+
let app = express(),
|
|
325
|
+
reloadPreview,
|
|
326
|
+
|
|
327
|
+
template;
|
|
328
|
+
|
|
329
|
+
// Editor and settings/bindings
|
|
330
|
+
app.get("/", function (req, res) {
|
|
331
|
+
log.success("Loading main page in browser");
|
|
332
|
+
res.header("Content-Type", "text/html; charset=utf-8")
|
|
333
|
+
.send(template.sdk_rendered);
|
|
334
|
+
});
|
|
335
|
+
|
|
336
|
+
app.get("/template.js", function (req, res) {
|
|
337
|
+
res.header("Content-Type", "application/javascript").send(template.template_js);
|
|
338
|
+
});
|
|
339
|
+
|
|
340
|
+
app.get("/template.js.map", function (req, res) {
|
|
341
|
+
res.sendFile(path.resolve(template_dir, "template.js.map"));
|
|
342
|
+
});
|
|
343
|
+
|
|
344
|
+
app.get("/comms.js", function (req, res) {
|
|
345
|
+
res.header("Content-Type", "application/javascript").send(comms_js.withoutOriginCheck + comms_js.validate);
|
|
346
|
+
});
|
|
347
|
+
|
|
348
|
+
app.get("/thumbnail", function (req, res) {
|
|
349
|
+
const jpg_path = path.resolve(template_dir, "thumbnail.jpg"),
|
|
350
|
+
png_path = path.resolve(template_dir, "thumbnail.png");
|
|
351
|
+
if (fs.existsSync(jpg_path)) {
|
|
352
|
+
return res.header("Content-Type", "image/jpeg").sendFile(jpg_path);
|
|
353
|
+
}
|
|
354
|
+
if (fs.existsSync(png_path)) {
|
|
355
|
+
return res.header("Content-Type", "image/jpeg").sendFile(png_path);
|
|
356
|
+
}
|
|
357
|
+
return res.status(404).send("thumbnail not found");
|
|
358
|
+
});
|
|
359
|
+
|
|
360
|
+
app.get("/template/1/embed/", function(req, res) {
|
|
361
|
+
res.header("Content-Type", "text/html; charset=utf-8")
|
|
362
|
+
.send(template.template_rendered);
|
|
363
|
+
});
|
|
364
|
+
|
|
365
|
+
// API for accessing data tables
|
|
366
|
+
app.get("/api/data_table/:id/csv", function(req, res) {
|
|
367
|
+
res.status(200).header("Content-Type", "text/csv")
|
|
368
|
+
.sendFile(path.resolve(template_dir, "data", req.params.id + ".csv"));
|
|
369
|
+
});
|
|
370
|
+
|
|
371
|
+
// Preview not in an iframe
|
|
372
|
+
app.get("/preview", function(req, res) {
|
|
373
|
+
res.header("Content-Type", "text/html; charset=utf-8")
|
|
374
|
+
.send(template.preview_rendered);
|
|
375
|
+
});
|
|
376
|
+
app.use(`/${preview_static_prefix}/`, express.static(path.join(template_dir, "static")));
|
|
377
|
+
|
|
378
|
+
// Static files
|
|
379
|
+
app.use("/", express.static(path.join(__dirname, "..", "site")));
|
|
380
|
+
app.use(`/template/1/embed/${static_prefix}/`, express.static(path.join(template_dir, "static")));
|
|
381
|
+
|
|
382
|
+
|
|
383
|
+
function startServer(sdk_template, template_) {
|
|
384
|
+
template = template_;
|
|
385
|
+
|
|
386
|
+
// Run the server
|
|
387
|
+
const listen_hostname = options.listen || "localhost";
|
|
388
|
+
const server = app.listen(options.port, listen_hostname, function() {
|
|
389
|
+
const url = "http://" + listen_hostname + ":" + options.port + "/";
|
|
390
|
+
log.info(`Running server at ${url}`);
|
|
391
|
+
|
|
392
|
+
// Set up the WebSocket server and the reloadPreview() function
|
|
393
|
+
const sockets = new Set();
|
|
394
|
+
const websocket_server = new ws.Server({ server });
|
|
395
|
+
websocket_server.on("connection", function(socket) {
|
|
396
|
+
sockets.add(socket);
|
|
397
|
+
socket.on("close", function() { sockets.delete(socket); });
|
|
398
|
+
});
|
|
399
|
+
reloadPreview = function() {
|
|
400
|
+
for (let socket of sockets) socket.close();
|
|
401
|
+
};
|
|
402
|
+
|
|
403
|
+
watchForChanges(sdk_template);
|
|
404
|
+
if (options.open) tryToOpen(url);
|
|
405
|
+
})
|
|
406
|
+
.on("error", function(error) {
|
|
407
|
+
if (error.code === "EADDRINUSE") {
|
|
408
|
+
log.die("Another process is already listening on port " + options.port,
|
|
409
|
+
"Perhaps you’re already running flourish in another terminal?",
|
|
410
|
+
"You can use the --port option to listen on a different port");
|
|
411
|
+
}
|
|
412
|
+
log.die("Failed to start server", error.message);
|
|
413
|
+
});
|
|
414
|
+
}
|
|
415
|
+
|
|
416
|
+
let build_failed = new Set(),
|
|
417
|
+
rebuilding = new Set();
|
|
418
|
+
function watchForChanges(sdk_template) {
|
|
419
|
+
// Watch for file changes. If something changes, tell the page to reload itself
|
|
420
|
+
// If the source code has changed, rebuild it.
|
|
421
|
+
|
|
422
|
+
let reload_timer = null;
|
|
423
|
+
function reloadTemplate() {
|
|
424
|
+
if (rebuilding.size > 0) {
|
|
425
|
+
log.info("Not reloading template while rebuild is in progress.");
|
|
426
|
+
return;
|
|
427
|
+
}
|
|
428
|
+
if (reload_timer) {
|
|
429
|
+
clearTimeout(reload_timer);
|
|
430
|
+
reload_timer = null;
|
|
431
|
+
}
|
|
432
|
+
reload_timer = setTimeout(_reloadTemplate, 50);
|
|
433
|
+
}
|
|
434
|
+
function _reloadTemplate() {
|
|
435
|
+
reload_timer = null;
|
|
436
|
+
log.info("Reloading...");
|
|
437
|
+
loadTemplate(template_dir, sdk_template, build_failed)
|
|
438
|
+
.then((template_) => {
|
|
439
|
+
template = template_;
|
|
440
|
+
log.info("Template reloaded. Trying to reload preview.");
|
|
441
|
+
reloadPreview();
|
|
442
|
+
})
|
|
443
|
+
.catch((error) => {
|
|
444
|
+
log.problem("Failed to reload template", error.message);
|
|
445
|
+
});
|
|
446
|
+
}
|
|
447
|
+
|
|
448
|
+
// Run any custom watchers
|
|
449
|
+
if (template.build_rules) {
|
|
450
|
+
for (const build_rule of template.build_rules) {
|
|
451
|
+
if ("watch" in build_rule) {
|
|
452
|
+
const command_parts = shell_quote.parse(build_rule.watch),
|
|
453
|
+
prog = command_parts[0],
|
|
454
|
+
args = command_parts.slice(1);
|
|
455
|
+
|
|
456
|
+
const env = process.env;
|
|
457
|
+
env.NODE_ENV = "development";
|
|
458
|
+
|
|
459
|
+
log.info(`Running watcher command: ${build_rule.watch}`);
|
|
460
|
+
cross_spawn.spawn(prog, args, { cwd: template_dir, stdio: "inherit", env });
|
|
461
|
+
}
|
|
462
|
+
}
|
|
463
|
+
}
|
|
464
|
+
|
|
465
|
+
const chokidar_opts = { ignoreInitial: true, disableGlobbing: true, cwd: template_dir };
|
|
466
|
+
chokidar.watch(".", chokidar_opts).on("all", function(event_type, filename) {
|
|
467
|
+
const path_parts = filename.split(path.sep);
|
|
468
|
+
|
|
469
|
+
let should_reload = false;
|
|
470
|
+
if (sdk.TEMPLATE_SPECIAL.has(path_parts[0])) {
|
|
471
|
+
if (rebuilding.size > 0) return log.warn(`Rebuild in progress, ignoring change to ${filename}`);
|
|
472
|
+
log.info("Detected change to file: " + filename);
|
|
473
|
+
should_reload = true;
|
|
474
|
+
}
|
|
475
|
+
|
|
476
|
+
const build_commands = new Map();
|
|
477
|
+
if (template.build_rules) {
|
|
478
|
+
for (const build_rule of template.build_rules) {
|
|
479
|
+
if ((build_rule.directory && isPrefix(splitPath(build_rule.directory), path_parts))
|
|
480
|
+
|| (build_rule.files && build_rule.files.indexOf(filename) != -1))
|
|
481
|
+
{
|
|
482
|
+
build_commands.set(build_rule.key, build_rule.script);
|
|
483
|
+
}
|
|
484
|
+
}
|
|
485
|
+
}
|
|
486
|
+
|
|
487
|
+
if (build_commands.size > 0) {
|
|
488
|
+
const build_commands_to_run = [];
|
|
489
|
+
for (const [key, command] of build_commands) {
|
|
490
|
+
if (rebuilding.has(key)) continue;
|
|
491
|
+
rebuilding.add(key);
|
|
492
|
+
if (reload_timer) {
|
|
493
|
+
clearTimeout(reload_timer);
|
|
494
|
+
reload_timer = null;
|
|
495
|
+
}
|
|
496
|
+
log.info("Detected change to file: " + filename, "Running build for " + key);
|
|
497
|
+
build_commands_to_run.push(
|
|
498
|
+
sdk.runBuildCommand(template_dir, command, "development")
|
|
499
|
+
.then(() => {
|
|
500
|
+
rebuilding.delete(key);
|
|
501
|
+
build_failed.delete(key);
|
|
502
|
+
}, (error) => {
|
|
503
|
+
rebuilding.delete(key);
|
|
504
|
+
build_failed.add(key);
|
|
505
|
+
throw error;
|
|
506
|
+
})
|
|
507
|
+
);
|
|
508
|
+
}
|
|
509
|
+
|
|
510
|
+
Promise.all(build_commands_to_run)
|
|
511
|
+
.then(() => {
|
|
512
|
+
if (rebuilding.size == 0) {
|
|
513
|
+
log.success("Build process complete.");
|
|
514
|
+
reloadTemplate();
|
|
515
|
+
}
|
|
516
|
+
})
|
|
517
|
+
.catch((error) => {
|
|
518
|
+
if (build_failed.size > 0 && rebuilding.size == 0) {
|
|
519
|
+
reloadTemplate(); // To pass the build_failed flags
|
|
520
|
+
}
|
|
521
|
+
});
|
|
522
|
+
}
|
|
523
|
+
else if (should_reload) reloadTemplate();
|
|
524
|
+
});
|
|
525
|
+
}
|
|
526
|
+
|
|
527
|
+
loadSDKTemplate()
|
|
528
|
+
.then((sdk_template) => {
|
|
529
|
+
return Promise.all([
|
|
530
|
+
sdk_template, loadTemplate(template_dir, sdk_template)
|
|
531
|
+
]);
|
|
532
|
+
})
|
|
533
|
+
.then(([sdk_template, template]) => {
|
|
534
|
+
startServer(sdk_template, template);
|
|
535
|
+
})
|
|
536
|
+
.catch((error) => {
|
|
537
|
+
if (options.debug) log.problem("Failed to start server", error.message, error.stack);
|
|
538
|
+
else log.problem("Failed to start server", error.message);
|
|
539
|
+
});
|
|
540
|
+
};
|