@flourish/sdk 3.17.3 → 3.18.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@flourish/sdk",
-  "version": "3.17.3",
+  "version": "3.18.0",
   "description": "The Flourish SDK",
   "module": "src/index.js",
   "scripts": {

package/package.json~

@@ -0,0 +1,53 @@
+{
+  "name": "@flourish/sdk",
+<<<<<<< HEAD
+  "version": "3.13.1",
+=======
+  "version": "3.14.0",
+>>>>>>> sdk-watch-command
+  "description": "The Flourish SDK",
+  "module": "src/index.js",
+  "scripts": {
+    "prepare": "cd .. && make sdk_clean sdk",
+    "test": "mocha"
+  },
+  "bin": {
+    "flourish": "bin/flourish"
+  },
+  "author": "Kiln Enterprises Ltd",
+  "license": "SEE LICENSE IN LICENSE.md",
+  "repository": "kiln/flourish-sdk",
+  "dependencies": {
+    "@flourish/semver": "^1.0.1",
+    "archiver": "^3.0.0",
+    "chokidar": "^3.0.1",
+    "colors": "^1.3.3",
+    "cross-spawn": "^6.0.5",
+    "d3-dsv": "^1.1.1",
+    "express": "^4.17.1",
+    "handlebars": "^4.1.2",
+    "js-yaml": "^3.13.1",
+    "minimist": "^1.2.0",
+    "ncp": "^2.0.0",
+    "parse5": "^5.1.0",
+    "read": "^1.0.7",
+    "request": "^2.88.0",
+    "resolve": "^1.11.1",
+    "rewrite-links": "^1.1.0",
+    "shell-quote": "^1.6.1",
+    "tmp": "^0.1.0",
+    "ws": "^7.0.1"
+  },
+  "devDependencies": {
+    "d3-request": "^1.0.6",
+    "mocha": "^6.1.4",
+    "rollup": "^1.16.2",
+    "rollup-plugin-node-resolve": "^5.1.0",
+    "rollup-plugin-uglify": "^6.0.2",
+    "sinon": "^8.1.1",
+    "tempy": "^0.3.0"
+  },
+  "engines": {
+    "node": ">=8.3"
+  }
+}

package/server/index.js~

@@ -0,0 +1,540 @@
+"use strict";
+
+// Modules
+const crypto = require("crypto"),
+      fs = require("fs"),
+      path = require("path"),
+
+      cross_spawn = require("cross-spawn"),
+      chokidar = require("chokidar"),
+      d3_dsv = require("d3-dsv"),
+      express = require("express"),
+      handlebars = require("handlebars"),
+      shell_quote = require("shell-quote"),
+      ws = require("ws"),
+      yaml = require("js-yaml"),
+
+      columns = require("./columns"),
+      comms_js = require("./comms_js"),
+      data_utils = require("./data"),
+      index_html = require("./index_html"),
+      json = require("./json"),
+
+      log = require("../lib/log"),
+      sdk = require("../lib/sdk");
+
+const TA = require("parse5/lib/tree-adapters/default.js");
+
+// Generate a static prefix randomly
+//
+// Use a different prefix for /preview, to catch the situation where the template
+//   developer mistakenly prepends a / to the static prefix.
+const static_prefix = crypto.randomBytes(15).toString("base64").replace(/[+/]/g, (c) => ({ "/": "_", "+": "-" })[c]),
+      preview_static_prefix = crypto.randomBytes(15).toString("base64").replace(/[+/]/g, (c) => ({ "/": "_", "+": "-" })[c]);
+
+function loadFile(path_parts, options) {
+	return new Promise(function(resolve, reject) {
+		const file_path = path.join(...path_parts),
+		      filename = path_parts[path_parts.length - 1];
+
+		function fail(message, error) {
+			if ("default" in options) {
+				log.warn(message, `Proceeding without ${filename}...`);
+				if (typeof options.default === "function") {
+					return resolve(options.default());
+				}
+				return resolve(options.default);
+			}
+			log.problem(message, error.message);
+			reject(error);
+		}
+
+		function succeed(result) {
+			if (!options.silentSuccess) log.success(`Loaded ${filename}`);
+			resolve(result);
+		}
+
+		fs.readFile(file_path, "utf8", function(error, loaded_text) {
+			if (error) return fail(`Failed to load ${file_path}`, error);
+			switch (options.type) {
+				case "json":
+					try { return succeed(JSON.parse(loaded_text)); }
+					catch (error) {
+						return fail(`Uh-oh! There's a problem with your ${filename} file.`, error);
+					}
+
+				case "yaml":
+					try { return succeed(yaml.safeLoad(loaded_text)); }
+					catch (error) {
+						return fail(`Uh-oh! There's a problem with your ${filename} file.`, error);
+					}
+
+				default:
+					return succeed(loaded_text);
+			}
+		});
+	});
+}
+
+function loadSDKTemplate() {
+	return loadFile([__dirname, "views", "index.html"], { silentSuccess: true })
+		.then((template_text) => handlebars.compile(template_text));
+}
+
+function loadTemplateText(template_dir) {
+	return loadFile([template_dir, "index.html"], {
+		default: () => loadFile([__dirname, "views", "default_template_index.html"], {
+			silentSuccess: true
+		})
+	});
+}
+
+function loadJavaScript(template_dir) {
+	return loadFile([template_dir, "template.js"], {});
+}
+
+function loadSettings(template_dir) {
+	return sdk.readAndValidateConfig(template_dir);
+}
+
+function listDataTables(template_dir) {
+	return new Promise(function(resolve, reject) {
+		fs.readdir(path.join(template_dir, "data"), function(error, filenames) {
+			if (error) {
+				if (error.code === "ENOENT") return resolve([]);
+				return reject(error);
+			}
+
+			const data_files = [];
+			for (let filename of filenames) {
+				if (!filename.endsWith(".csv")) continue;
+
+				var name = filename.substr(0, filename.length - 4);
+				data_files.push(name);
+			}
+			resolve(data_files);
+		});
+	});
+}
+
+function getData(template_dir, data_tables) {
+	return Promise.all(data_tables.map((data_table) => getDataTable(template_dir, data_table)))
+		.then((data_array) => {
+			const data_by_name = {};
+			const column_types_by_name = {};
+			for (var i = 0; i < data_tables.length; i++) {
+				data_by_name[data_tables[i]] = data_array[i];
+			}
+			for (const data_table in data_by_name) {
+				const data = data_by_name[data_table];
+				column_types_by_name[data_table] = data_utils.transposeNestedArray(data)
+					.map(function(d, i) { return { type_id: data_utils.interpretColumn(d)[0].id, index: i }; });
+			}
+			return { data: data_by_name, column_types_by_name };
+		});
+}
+
+function getDataTable(template_dir, data_table) {
+	return new Promise(function(resolve, reject) {
+		fs.readFile(path.join(template_dir, "data", data_table + ".csv"), "utf8", function(error, csv_text) {
+			if (error) return reject(error);
+			if (csv_text.charAt(0) === "\uFEFF") csv_text = csv_text.substr(1);
+			resolve(d3_dsv.csvParseRows(csv_text));
+		});
+	});
+}
+
+function parseDataBindings(data_bindings, data_tables) {
+	if (!data_bindings) return {1: {}};
+
+	// Use the names as ids
+	const name_by_id = {};
+	for (let name of data_tables) name_by_id[name] = name;
+
+	// Collect parsed bindings by dataset
+	const data_bindings_by_dataset = {};
+	for (let binding of data_bindings) {
+		let dataset = binding.dataset;
+		if (!dataset) continue;
+
+		if (!data_bindings_by_dataset[dataset]) data_bindings_by_dataset[dataset] = {};
+		data_bindings_by_dataset[dataset][binding.key] = columns.parseDataBinding(binding, name_by_id);
+	}
+	return { 1: data_bindings_by_dataset };
+}
+
+function documentFragment(elements) {
+	const fragment = TA.createDocumentFragment();
+	for (const element of elements) {
+		TA.appendChild(fragment, element);
+	}
+	return fragment;
+}
+
+function scriptElementInline(code) {
+	const element = TA.createElement("script", "http://www.w3.org/1999/xhtml", []);
+	TA.insertText(element, code);
+	return element;
+}
+
+function scriptElementExternal(url) {
+	return TA.createElement("script", "http://www.w3.org/1999/xhtml", [{ name: "src", value: url }]);
+}
+
+
+function loadTemplate(template_dir, sdk_template, build_failed) {
+	return Promise.all([
+		listDataTables(template_dir),
+		loadSettings(template_dir),
+	])
+	.then(([data_tables, settings]) => {
+		const data_bindings = parseDataBindings(settings.data, data_tables);
+		return Promise.all([
+			settings, data_bindings, data_tables,
+			previewInitJs(template_dir, data_bindings["1"], data_tables),
+			loadTemplateText(template_dir),
+			loadJavaScript(template_dir)
+		]);
+	})
+	.then(([
+		settings, data_bindings, data_tables,
+		preview_init_js, template_text, template_js
+	]) => {
+		const page_params = {
+			// Always use ID of 1 for SDK
+			visualisation: { id: 1, can_edit: true },
+			visualisation_js: "new Flourish.Visualisation('1', 0," + json.safeStringify({
+				data_bindings: data_bindings,
+				data_tables: data_tables,
+			}) + ")",
+			settings: json.safeStringify(settings.settings || []),
+			data_bindings: json.safeStringify(settings.data || []),
+			template_name: settings.name || "Untitled template",
+			template_version: settings.version,
+			template_author: settings.author || "",
+			build_failed: build_failed && build_failed.size > 0
+		};
+
+		const script = documentFragment([
+			scriptElementInline("window.Flourish = " + json.safeStringify({
+				static_prefix, environment: "sdk"
+			}) + ";"),
+			scriptElementExternal("/template.js"),
+			scriptElementExternal("/comms.js"),
+			scriptElementExternal("/embedded.js"),
+		]);
+
+		const preview_script = documentFragment([
+			scriptElementInline("window.Flourish = " + json.safeStringify({
+				static_prefix: preview_static_prefix, environment: "sdk"
+			}) + ";"),
+			scriptElementExternal("/template.js"),
+			scriptElementExternal("/comms.js"),
+			scriptElementExternal("/embedded.js"),
+			scriptElementExternal("/talk_to_server.js"),
+			scriptElementInline("_Flourish_talkToServer();"),
+			scriptElementInline(preview_init_js),
+		]);
+
+		return Promise.all([
+			sdk_template(page_params),
+			index_html.render(template_text, {
+				title: "Flourish SDK template preview blah",
+				static: static_prefix,
+				parsed_script: script
+			}),
+			index_html.render(template_text, {
+				title: "Flourish SDK template preview flerm",
+				static: preview_static_prefix,
+				parsed_script: preview_script,
+			}),
+			template_js,
+			sdk.buildRules(template_dir),
+		]);
+	})
+	.then(([sdk_rendered, template_rendered, preview_rendered, template_js, build_rules]) => ({
+		sdk_rendered, template_rendered, preview_rendered, template_js, build_rules
+	}));
+}
+
+function previewInitJs(template_dir, data_bindings, data_tables) {
+	return getData(template_dir, data_tables).then(({data, column_types_by_name}) => {
+		const prepared_data = {};
+		for (let dataset in data_bindings) {
+			prepared_data[dataset] = data_utils.extractData(data_bindings[dataset], data, column_types_by_name);
+		}
+
+		const column_names = {};
+		const type_information = {};
+		for (let dataset in prepared_data) {
+			column_names[dataset] = prepared_data[dataset].column_names;
+			type_information[dataset] = prepared_data[dataset].type_information;
+		}
+
+		return `
+		var _Flourish_data_column_names = ${json.safeStringify(column_names)},
+		    _Flourish_data_type_information = ${json.safeStringify(type_information)},
+		    _Flourish_data = ${json.javaScriptStringify(prepared_data)};
+		for (var _Flourish_dataset in _Flourish_data) {
+			window.template.data[_Flourish_dataset] = _Flourish_data[_Flourish_dataset];
+			window.template.data[_Flourish_dataset].column_names = _Flourish_data_column_names[_Flourish_dataset];
+			window.template.data[_Flourish_dataset].type_information = _Flourish_data_type_information[_Flourish_dataset];
+		}
+		window.template.draw();
+		`;
+	});
+}
+
+function tryToOpen(url) {
+	// If it’s available and works, use /usr/bin/open to open
+	// the URL. If not just prompt the user to open it.
+	try {
+		cross_spawn.spawn("/usr/bin/open", [url])
+			.on("exit", function(exit_code) {
+				if (exit_code != 0) {
+					log.success("Now open " + url + " in your web browser!");
+				}
+				else {
+					log.success("Opened browser window to " + url);
+				}
+			})
+			.on("error", function() {
+				log.success("Now open " + url + " in your web browser!");
+			});
+	}
+	catch (error) {
+		log.success("Now open " + url + " in your web browser!");
+	}
+}
+
+function isPrefix(a, b) {
+	if (a.length > b.length) return false;
+	for (let i = 0; i < a.length; i++) {
+		if (a[i] !== b[i]) return false;
+	}
+	return true;
+}
+
+function splitPath(p) {
+	return p.split(path.sep).filter(c => c != "");
+}
+
+
+module.exports = function(template_dir, options) {
+	let app = express(),
+	    reloadPreview,
+
+	    template;
+
+	// Editor and settings/bindings
+	app.get("/", function (req, res) {
+		log.success("Loading main page in browser");
+		res.header("Content-Type", "text/html; charset=utf-8")
+			.send(template.sdk_rendered);
+	});
+
+	app.get("/template.js", function (req, res) {
+		res.header("Content-Type", "application/javascript").send(template.template_js);
+	});
+
+	app.get("/template.js.map", function (req, res) {
+		res.sendFile(path.resolve(template_dir, "template.js.map"));
+	});
+
+	app.get("/comms.js", function (req, res) {
+		res.header("Content-Type", "application/javascript").send(comms_js.withoutOriginCheck + comms_js.validate);
+	});
+
+	app.get("/thumbnail", function (req, res) {
+		const jpg_path = path.resolve(template_dir, "thumbnail.jpg"),
+		      png_path = path.resolve(template_dir, "thumbnail.png");
+		if (fs.existsSync(jpg_path)) {
+			return res.header("Content-Type", "image/jpeg").sendFile(jpg_path);
+		}
+		if (fs.existsSync(png_path)) {
+			return res.header("Content-Type", "image/jpeg").sendFile(png_path);
+		}
+		return res.status(404).send("thumbnail not found");
+	});
+
+	app.get("/template/1/embed/", function(req, res) {
+		res.header("Content-Type", "text/html; charset=utf-8")
+			.send(template.template_rendered);
+	});
+
+	// API for accessing data tables
+	app.get("/api/data_table/:id/csv", function(req, res) {
+		res.status(200).header("Content-Type", "text/csv")
+			.sendFile(path.resolve(template_dir, "data", req.params.id + ".csv"));
+	});
+
+	// Preview not in an iframe
+	app.get("/preview", function(req, res) {
+		res.header("Content-Type", "text/html; charset=utf-8")
+			.send(template.preview_rendered);
+	});
+	app.use(`/${preview_static_prefix}/`, express.static(path.join(template_dir, "static")));
+
+	// Static files
+	app.use("/", express.static(path.join(__dirname, "..", "site")));
+	app.use(`/template/1/embed/${static_prefix}/`, express.static(path.join(template_dir, "static")));
+
+
+	function startServer(sdk_template, template_) {
+		template = template_;
+
+		// Run the server
+		const listen_hostname = options.listen || "localhost";
+		const server = app.listen(options.port, listen_hostname, function() {
+			const url = "http://" + listen_hostname + ":" + options.port + "/";
+			log.info(`Running server at ${url}`);
+
+			// Set up the WebSocket server and the reloadPreview() function
+			const sockets = new Set();
+			const websocket_server = new ws.Server({ server });
+			websocket_server.on("connection", function(socket) {
+				sockets.add(socket);
+				socket.on("close", function() { sockets.delete(socket); });
+			});
+			reloadPreview = function() {
+				for (let socket of sockets) socket.close();
+			};
+
+			watchForChanges(sdk_template);
+			if (options.open) tryToOpen(url);
+		})
+		.on("error", function(error) {
+			if (error.code === "EADDRINUSE") {
+				log.die("Another process is already listening on port " + options.port,
+					"Perhaps you’re already running flourish in another terminal?",
+					"You can use the --port option to listen on a different port");
+			}
+			log.die("Failed to start server", error.message);
+		});
+	}
+
+	let build_failed = new Set(),
+	    rebuilding = new Set();
+	function watchForChanges(sdk_template) {
+		// Watch for file changes. If something changes, tell the page to reload itself
+		// If the source code has changed, rebuild it.
+
+		let reload_timer = null;
+		function reloadTemplate() {
+			if (rebuilding.size > 0) {
+				log.info("Not reloading template while rebuild is in progress.");
+				return;
+			}
+			if (reload_timer) {
+				clearTimeout(reload_timer);
+				reload_timer = null;
+			}
+			reload_timer = setTimeout(_reloadTemplate, 50);
+		}
+		function _reloadTemplate() {
+			reload_timer = null;
+			log.info("Reloading...");
+			loadTemplate(template_dir, sdk_template, build_failed)
+				.then((template_) => {
+					template = template_;
+					log.info("Template reloaded. Trying to reload preview.");
+					reloadPreview();
+				})
+				.catch((error) => {
+					log.problem("Failed to reload template", error.message);
+				});
+		}
+
+		// Run any custom watchers
+		if (template.build_rules) {
+			for (const build_rule of template.build_rules) {
+				if ("watch" in build_rule) {
+					const command_parts = shell_quote.parse(build_rule.watch),
+					      prog = command_parts[0],
+					      args = command_parts.slice(1);
+
+					const env = process.env;
+					env.NODE_ENV = "development";
+
+					log.info(`Running watcher command: ${build_rule.watch}`);
+					cross_spawn.spawn(prog, args, { cwd: template_dir, stdio: "inherit", env });
+				}
+			}
+		}
+
+		const chokidar_opts = { ignoreInitial: true, disableGlobbing: true, cwd: template_dir };
+		chokidar.watch(".", chokidar_opts).on("all", function(event_type, filename) {
+			const path_parts = filename.split(path.sep);
+
+			let should_reload = false;
+			if (sdk.TEMPLATE_SPECIAL.has(path_parts[0])) {
+				if (rebuilding.size > 0) return log.warn(`Rebuild in progress, ignoring change to ${filename}`);
+				log.info("Detected change to file: " + filename);
+				should_reload = true;
+			}
+
+			const build_commands = new Map();
+			if (template.build_rules) {
+				for (const build_rule of template.build_rules) {
+					if ((build_rule.directory && isPrefix(splitPath(build_rule.directory), path_parts))
+						|| (build_rule.files && build_rule.files.indexOf(filename) != -1))
+					{
+						build_commands.set(build_rule.key, build_rule.script);
+					}
+				}
+			}
+
+			if (build_commands.size > 0) {
+				const build_commands_to_run = [];
+				for (const [key, command] of build_commands) {
+					if (rebuilding.has(key)) continue;
+					rebuilding.add(key);
+					if (reload_timer) {
+						clearTimeout(reload_timer);
+						reload_timer = null;
+					}
+					log.info("Detected change to file: " + filename, "Running build for " + key);
+					build_commands_to_run.push(
+						sdk.runBuildCommand(template_dir, command, "development")
+							.then(() => {
+								rebuilding.delete(key);
+								build_failed.delete(key);
+							}, (error) => {
+								rebuilding.delete(key);
+								build_failed.add(key);
+								throw error;
+							})
+					);
+				}
+
+				Promise.all(build_commands_to_run)
+					.then(() => {
+						if (rebuilding.size == 0) {
+							log.success("Build process complete.");
+							reloadTemplate();
+						}
+					})
+					.catch((error) => {
+						if (build_failed.size > 0 && rebuilding.size == 0) {
+							reloadTemplate(); // To pass the build_failed flags
+						}
+					});
+			}
+			else if (should_reload) reloadTemplate();
+		});
+	}
+
+	loadSDKTemplate()
+		.then((sdk_template) => {
+			return Promise.all([
+				sdk_template, loadTemplate(template_dir, sdk_template)
+			]);
+		})
+		.then(([sdk_template, template]) => {
+			startServer(sdk_template, template);
+		})
+		.catch((error) => {
+			if (options.debug) log.problem("Failed to start server", error.message, error.stack);
+			else log.problem("Failed to start server", error.message);
+		});
+};

package/test/lib/validate_config.js

@@ -916,12 +916,21 @@ describe("validate_config", function() {
 						"template.yml Setting import overrides must be an array"
 					);
 				});
-				it("should throw if an override is missing the 'property' property", function() {
+				it("should throw if an override is missing the 'property' or 'tag' property", function() {
 					expectFailure(
 						metadataPlus({ settings: [{ property: "imported_prop", import: "@flourish/layout", overrides: [{}] }]}),
-						`template.yml Setting import overrides must each specify overridden “property”`
+						`template.yml Setting import overrides must each specify overridden “property” or “tag”`
 					);
 				});
+				it("should throw if an override has both 'property' and 'tag' property", function () {
+					expectFailure(
+						metadataPlus({ settings: [{ property: "imported_prop", import: "@flourish/layout", overrides: [{property: "bg_color", tag: "categorical"}] }] }),
+						`template.yml Setting import overrides cannot contain both “property” and “tag” property`
+					);
+				});
+				it("should allow for an override to have 'tag' in place of 'property'", function () {
+					expectSuccess(metadataPlus({ settings: [{ property: "imported_prop", import: "@flourish/layout", overrides: [{ tag: "categorical" }] }] }));
+				});
 				it("should allow for an override without a 'method' property", function() {
 					expectSuccess(metadataPlus({ settings: [{ property: "imported_prop", import: "@flourish/layout", overrides: [{property: "bg_color"}] }]}));
 				});