@flourish/sdk 3.16.0 → 3.17.0

package/RELEASE_NOTES.md

@@ -1,3 +1,7 @@
+# 3.17.0
+
+* Allow published visualisations to be cloned into the SDK
+
 # 3.16.0
 * Upgrade all NPM dependencies.
 

package/audit-resolve.json

@@ -0,0 +1,36 @@
+{
+  "decisions": {
+    "1002401|npm-audit-resolver>yargs-unparser>yargs>string-width>strip-ansi>ansi-regex": {
+      "decision": "ignore",
+      "madeAt": 1637059117447,
+      "expiresAt": 1637663913365
+    },
+    "1002401|npm-audit-resolver>yargs-unparser>yargs>cliui>string-width>strip-ansi>ansi-regex": {
+      "decision": "ignore",
+      "madeAt": 1637059117447,
+      "expiresAt": 1637663913365
+    },
+    "1002401|npm-audit-resolver>yargs-unparser>yargs>cliui>wrap-ansi>string-width>strip-ansi>ansi-regex": {
+      "decision": "ignore",
+      "madeAt": 1637059117447,
+      "expiresAt": 1637663913365
+    },
+    "1004946|npm-audit-resolver>yargs-unparser>yargs>string-width>strip-ansi>ansi-regex": {
+      "decision": "ignore",
+      "madeAt": 1639560165214,
+      "expiresAt": 1642152161687
+    },
+    "1004946|npm-audit-resolver>yargs-unparser>yargs>cliui>string-width>strip-ansi>ansi-regex": {
+      "decision": "ignore",
+      "madeAt": 1639560165214,
+      "expiresAt": 1642152161687
+    },
+    "1004946|npm-audit-resolver>yargs-unparser>yargs>cliui>wrap-ansi>string-width>strip-ansi>ansi-regex": {
+      "decision": "ignore",
+      "madeAt": 1639560165214,
+      "expiresAt": 1642152161687
+    }
+  },
+  "rules": {},
+  "version": 1
+}

package/lib/cmd/assign-version-number.js

@@ -9,7 +9,7 @@ function assign_version_number(args, server_opts) {
 	let template_id_promise, template_version;
 	if (args._.length == 2) {
 		// Assume the supplied argument is a version number, and try to get the id from the current directory
-		template_id_promise = sdk.readAndValidateConfig(".").then(config => config.id);
+		template_id_promise = sdk.readAndValidateConfig(".").then(({config}) => config.id);
 		template_version = args._[1];
 	}
 	else if (args._.length == 3) {

package/lib/cmd/help.js

@@ -24,7 +24,7 @@ help.help = `
 Commands:
 	flourish assign-version-number [template id] version
 	flourish build [build rules...]
-	flourish delete [--force] template_id
+	flourish delete [--force] template_id version
 	flourish [-h|--help|help] [topic]
 	flourish history [--full] template_id
 	flourish list [--full] [template id]

package/lib/cmd/publish.js

@@ -5,9 +5,13 @@ var fs = require("fs"),
 
     archiver = require("archiver"),
     tmp = require("tmp"),
+    FormData = require("form-data"),
+
+    d3_dsv = require("d3-dsv"),
 
     log = require("../log"),
-    sdk = require("../sdk");
+    sdk = require("../sdk"),
+    data_utils = require("../../server/data");
 
 function zipUpTemplate(template_dir, config) {
 	return new Promise(function(resolve, reject) {
@@ -48,6 +52,21 @@ function zipUpTemplate(template_dir, config) {
 			if (config.settings) zip.append(JSON.stringify(config.settings), { name: "settings.js" });
 			if (config.data) zip.append(JSON.stringify(config.data), { name: "data.json" });
 
+			const data_dir = path.join(template_dir, "data");
+			if (fs.existsSync(data_dir)) {
+				// FIXME: check inferred types are compatible with specified types of data bindings
+				const column_types_by_sheet = {};
+				const files = fs.readdirSync(data_dir).filter(d => d.endsWith(".csv"));
+				for (const file of files) {
+					let csv_text = fs.readFileSync(path.join(template_dir, "data", file), "utf8");
+					if (csv_text.charAt(0) === "\uFEFF") csv_text = csv_text.substr(1);
+					const parsed_csv = d3_dsv.csvParseRows(csv_text);
+					column_types_by_sheet[file.replace(/\.csv$/, "")] = data_utils.getColumnTypesForData(parsed_csv);
+				}
+				zip.append(JSON.stringify(column_types_by_sheet), { name: "column_types_by_sheet.json" });
+			}
+
+
 			if (fs.existsSync(path.join(template_dir, "GUIDE.md"))) {
 				let file_path = path.join(template_dir, "GUIDE.md");
 				zip.file(file_path, { name: "README.md" });
@@ -77,17 +96,16 @@ function zipUpTemplate(template_dir, config) {
 }
 
 function uploadTemplate(server_opts, template_id, external_version, zip_filename) {
-	return sdk.request(server_opts, "template/publish", {
-		id: template_id,
-		version: external_version,
-		template: {
-			value: fs.createReadStream(zip_filename),
-			options: {
-				filename: "template.zip",
-				contentType: "application/zip",
-			}
-		}
+	const body = new FormData();
+
+	body.append("id", template_id);
+	body.append("version", external_version);
+	body.append("template", fs.createReadStream(zip_filename), {
+		contentType: "application/zip",
+		filename: "template.zip"
 	});
+
+	return sdk.request(server_opts, "template/publish", body);
 }
 
 function publish(args, server_opts) {
@@ -100,7 +118,7 @@ function publish(args, server_opts) {
 			if (args.release) return sdk.removePrereleaseTag(template_dir);
 		})
 		.then(() => sdk.readAndValidateConfig(template_dir))
-		.then((config) => {
+		.then(({config, warnings}) => {
 			if (!config.id) log.die("The template’s template.yml doesn’t have an id. Add one and try again.");
 
 			if (config.id.indexOf("/") > -1) {
@@ -143,6 +161,7 @@ function publish(args, server_opts) {
 						log.victory(`Uploaded version ${external_version} on ${dt.toDateString()} at ${dt.toTimeString()}`,
 							`Your template is available at ${protocol}://${server_opts.host}/@${template_path}/${external_version}`);
 					}
+					warnings.forEach(warning => log.warn(warning));
 				});
 		})
 		.catch((error) => {

package/lib/cmd/run.js

@@ -17,7 +17,8 @@ function run(args) {
 			port: port,
 			listen: args.listen,
 			open: args.open,
-			debug: args.debug
+			debug: args.debug,
+			host: args.host
 		});
 	}
 

package/lib/cmd/version.js

@@ -4,7 +4,7 @@ var log = require("../log"),
     sdk = require("../sdk");
 
 function version(args) {
-	sdk.getSDKVersion()
+	sdk.getSdkVersion()
 		.then((version_number) => console.log(version_number))
 		.catch((error) => {
 			if (args.debug) log.die("Failed to get SDK version number", error.message, error.stack);

package/lib/log.js

@@ -12,7 +12,7 @@ function info(...lines) {
 	for (let line of lines) console.log(colors.yellow("   " + line));
 }
 function warn(...lines) {
-	for (let line of lines) console.warn(colors.yellow("   " + line));
+	for (let line of lines) console.warn(colors.yellow("⚠️   " + line));
 }
 function warn_bold(...lines) {
 	for (let line of lines) console.warn(colors.bold.yellow("   " + line));

package/lib/sdk.js

@@ -4,7 +4,8 @@ const fs = require("fs"),
       path = require("path"),
 
       cross_spawn = require("cross-spawn"),
-      mod_request = require("request"),
+      fetch = require("node-fetch"),
+      FormData = require("form-data"),
       shell_quote = require("shell-quote"),
       yaml = require("js-yaml"),
       nodeResolve = require("resolve"),
@@ -21,7 +22,7 @@ const YAML_DUMP_OPTS = { flowLevel: 4 };
 
 const package_json_filename = path.join(__dirname, "..", "package.json");
 var sdk_version = null;
-function getSDKVersion() {
+function getSdkVersion() {
 	if (sdk_version) return Promise.resolve(sdk_version);
 	return new Promise(function(resolve, reject) {
 		fs.readFile(package_json_filename, "utf8", function(error, package_json) {
@@ -33,7 +34,7 @@ function getSDKVersion() {
 }
 
 function getSDKMajorVersion() {
-	return getSDKVersion()
+	return getSdkVersion()
 		.then((sdk_version) => {
 			const version_tuple = sdk_version.split(".").map((x) => parseInt(x));
 			return version_tuple[0];
@@ -94,77 +95,83 @@ const AUTHENTICATED_REQUEST_METHODS = new Set([
 	"user/whoami"
 ]);
 
-const MULTIPART_REQUEST_METHODS = new Set([
-	"template/publish",
-]);
+async function request(server_opts, method, data) {
+	let sdk_token;
 
-function request(server_opts, method, data) {
-	let read_sdk_token_if_necessary;
 	if (AUTHENTICATED_REQUEST_METHODS.has(method)) {
-		read_sdk_token_if_necessary = getSdkToken(server_opts)
-			.catch((error) => {
-				log.problem(`Failed to read ${sdk_tokens_file}`, error.message);
-			})
-			.then((sdk_token) => {
-				if (!sdk_token) {
-					log.die("You are not logged in. Try ‘flourish login’ or ‘flourish register’ first.");
-				}
-				return sdk_token;
-			});
-	}
-	else {
-		read_sdk_token_if_necessary = Promise.resolve();
+		try {
+			sdk_token = await getSdkToken(server_opts);
+		}
+		catch (error) {
+			log.problem(`Failed to read ${sdk_tokens_file}`, error.message);
+		}
+		if (!sdk_token) {
+			log.die("You are not logged in. Try ‘flourish login’ or ‘flourish register’ first.");
+		}
 	}
 
-	return Promise.all([read_sdk_token_if_necessary, getSDKVersion()])
-		.then(([sdk_token, sdk_version]) => new Promise(function(resolve, reject) {
-			let protocol = "https";
-			if (server_opts.host.match(/^(localhost|127\.0\.0\.1|.*\.local)(:\d+)?$/)) {
-				protocol = "http";
-			}
-			let url = protocol + "://" + server_opts.host + "/api/v1/" + method;
-			let request_params = {
-				method: "POST",
-				uri: url,
-			};
-
-			Object.assign(data, { sdk_token, sdk_version });
-			if (server_opts.user) {
-				request_params.auth = {
-					user: server_opts.user,
-					pass: server_opts.password,
-					sendImmediately: true,
-				};
-			}
+	const sdk_version = await getSdkVersion();
+	const protocol = server_opts.host.match(/^(localhost|127\.0\.0\.1|.*\.local)(:\d+)?$/) ? "http:" : "https:";
+	const url = `${protocol}//${server_opts.host}/api/v1/${method}`;
+	const options = { method: data ? "POST" : "GET" };
 
-			if (MULTIPART_REQUEST_METHODS.has(method)) {
-				request_params.formData = data;
-			}
-			else {
-				request_params.headers = { "Content-Type": "application/json" };
-				request_params.body = JSON.stringify(data);
+	if (data) {
+		if (data instanceof FormData) {
+			if (sdk_token) {
+				data.append("sdk_token", sdk_token);
 			}
+			data.append("sdk_version", sdk_version);
+			options.headers = data.getHeaders();
+			options.body = data;
+		}
+		else {
+			options.body = JSON.stringify({ ...data, sdk_token, sdk_version });
+			options.headers = { "content-type": "application/json" };
+		}
+	}
 
-			mod_request(request_params, function(error, res) {
-				if (error) log.die(error);
-				if (res.statusCode == 200) {
-					let r;
-					try { r = JSON.parse(res.body); }
-					catch (error) {
-						log.die("Failed to parse response from server", error, res.body);
-					}
-					return resolve(r);
-				}
+	if (server_opts.user) {
+		options.headers.authorization = Buffer.from(`${server_opts.user}:${server_opts.password}`).toString("base64");
+	}
 
-				// We got an error response. See if we can parse it to extract an error message
-				try {
-					let r = JSON.parse(res.body);
-					if ("error" in r) log.die("Error from server", r.error);
-				}
-				catch (e) { }
-				log.die("Server error", res.body);
-			});
-		}));
+	let res;
+
+	try {
+		res = await fetch(url, options);
+	}
+	catch (e) {
+		log.die(e);
+	}
+
+	let text;
+
+	try {
+		// We could use res.json() here, but we're interested in what the body
+		// is when it's *not* json (load balancer issues etc.).
+		text = await res.text();
+	}
+	catch (error) {
+		log.die("Failed to get response from server", error);
+	}
+
+	let body;
+
+	try {
+		body = JSON.parse(text);
+	}
+	catch (error) {
+		log.die("Failed to parse response body", res.status, error, text);
+	}
+
+	if (res.ok) {
+		return body;
+	}
+
+	if (body.error) {
+		log.die("Error from server", res.status, body.error);
+	}
+
+	log.die("Server error", res.status, body);
 }
 
 function runBuildCommand(template_dir, command, node_env) {
@@ -383,6 +390,26 @@ async function resolveImports(config, template_dir) {
 	return config;
 }
 
+// Sets a default binding data_type of string in templates with both typed and untyped bindings, and return a post-publish warning message.
+function checkDataTypes(config) {
+	const warnings = [];
+	if (!config.data) return { config, warnings };
+
+	const all_bindings = config.data.filter(binding => typeof binding !== "string"); // filter out title and description strings
+	const any_bindings_are_typed = all_bindings.some(binding => binding.data_type);
+
+	if (any_bindings_are_typed) {
+		config.data.forEach(binding => {
+			if (typeof binding === "string") return;
+			if (!binding.data_type) {
+				binding.data_type = "string";
+				warnings.push(`Missing data_type for key ${binding.key} in dataset ${binding.dataset} - assuming "string"`);
+			}
+		});
+	}
+	return { config, warnings };
+}
+
 function readAndValidateConfig(template_dir) {
 	return readConfig(template_dir)
 		.then((config) => {
@@ -390,7 +417,8 @@ function readAndValidateConfig(template_dir) {
 			return config;
 		})
 		.then(config => addShowConditions(config))
-		.then(config => resolveImports(config, template_dir));
+		.then(config => resolveImports(config, template_dir))
+		.then(config => checkDataTypes(config));
 }
 
 function changeVersionNumberInPackageJson(template_dir, change_function) {
@@ -531,7 +559,7 @@ const TEMPLATE_SPECIAL = new Set([
 ]);
 
 module.exports = {
-	checkTemplateVersion, getSDKVersion, getSDKMajorVersion,
+	checkTemplateVersion, getSdkVersion, getSDKMajorVersion,
 
 	getSdkToken, setSdkToken, deleteSdkTokens,
 	request,

package/lib/validate_config.js

@@ -227,7 +227,8 @@ function validateSetting(template_directory, conditional_settings, setting) {
 		throw new Error(`template.yml setting “${property}” has unsupported width property: must be “full”, “half”, “quarter” or “three quarters”`);
 	}
 	if ("size" in setting) {
-		if (setting.type !== "code" && setting.type !== "text") throw new Error(`template.yml setting “${property}” has a “size” setting but is not of type “string” or “code”`);
+		const can_set_size = setting.type == "code" || setting.type == "text" || (setting.type == "string" && setting.style == "buttons");
+		if (!can_set_size) throw new Error(`template.yml setting “${property}” has a “size” property; this requires type “text” or “code”, or type “string” with ”style: buttons”`);
 		else if (setting.size !== "large") throw new Error(`template.yml setting “${property}” has unsupported size property: must be “large”`);
 	}
 }
@@ -253,8 +254,8 @@ function validateSettings(template_directory, settings, bindings) {
 	if (conditional_settings.size > 0) {
 		conditional_settings.forEach(function(conditional_setting) {
 			if (/^data\./.test(conditional_setting)) {
-				if (!/^data\.\w+\.\w+$/.test(conditional_setting)) {
-					throw new Error(`template.yml: “show_if” or “hide_if” property specifies invalid data binding “${conditional_setting}”`);
+				if (!/^data\.\w+\.\w+(\.type)?$/.test(conditional_setting)) {
+					throw new Error(`template.yml: “show_if” or “hide_if” property specifies invalid data binding or column type “${conditional_setting}”`);
 				}
 				if (!bindings || !Array.isArray(bindings)) {
 					throw new Error(`template.yml: “show_if” or “hide_if” property refers to data binding “${conditional_setting}” when none are defined`);
@@ -271,7 +272,7 @@ function validateSettings(template_directory, settings, bindings) {
 	}
 }
 
-function validateColSpec(spec, parser, data_table_names) {
+function validateColSpec(spec, parser, data_table_names, is_optional) {
 	const double_colon_ix = spec.indexOf("::");
 	if (double_colon_ix == -1) throw new Error("Invalid data binding: " + spec);
 	const data_table_name = spec.substr(0, double_colon_ix);
@@ -280,7 +281,7 @@ function validateColSpec(spec, parser, data_table_names) {
 	}
 
 	const col_spec = spec.substr(double_colon_ix + 2);
-	parser(col_spec);
+	parser(col_spec, is_optional);
 }
 
 const VALID_DATA_BINDING_TYPES = new Set(["column", "columns"]);
@@ -320,7 +321,7 @@ function validateDataBinding(binding, data_table_names) {
 			if (typeof binding.column !== "string") {
 				throw new Error(`template.yml: “column” property of data binding “${binding_name}” must be a string`);
 			}
-			validateColSpec(binding.column, columns.parseColumn, data_table_names);
+			validateColSpec(binding.column, columns.parseColumn, data_table_names, binding.optional);
 		}
 	}
 	else if (binding.type == "columns") {

package/package.json

@@ -1,11 +1,13 @@
 {
   "name": "@flourish/sdk",
-  "version": "3.16.0",
+  "version": "3.17.0",
   "description": "The Flourish SDK",
   "module": "src/index.js",
   "scripts": {
     "prepare": "cd .. && make sdk_clean sdk",
-    "test": "mocha --recursive"
+    "test": "mocha --recursive",
+    "audit": "check-audit",
+    "audit:resolve": "resolve-audit"
   },
   "bin": {
     "flourish": "bin/flourish"
@@ -14,32 +16,37 @@
   "license": "SEE LICENSE IN LICENSE.md",
   "repository": "kiln/flourish-sdk",
   "dependencies": {
+    "@flourish/interpreter": "^6.0.3",
     "@flourish/semver": "^1.0.1",
+    "@flourish/transform-data": "^2.1.0",
     "@handlebars/allow-prototype-access": "^1.0.3",
+    "@rollup/plugin-commonjs": "^17.1.0",
     "archiver": "^5.0.2",
     "chokidar": "^3.4.3",
     "colors": "^1.4.0",
     "cross-spawn": "^7.0.3",
     "d3-dsv": "^2.0.0",
     "express": "^4.17.1",
+    "form-data": "^4.0.0",
     "handlebars": "^4.7.6",
     "js-yaml": "^3.14.0",
     "minimist": "^1.2.5",
     "ncp": "^2.0.0",
+    "node-fetch": "^2.6.6",
     "parse5": "^6.0.1",
     "read": "^1.0.7",
-    "request": "^2.88.2",
     "resolve": "^1.18.1",
     "rewrite-links": "^1.1.0",
     "rollup-plugin-terser": "^7.0.2",
     "shell-quote": "^1.7.2",
     "tmp": "^0.2.1",
-    "ws": "^7.3.1"
+    "ws": "^7.4.6"
   },
   "devDependencies": {
     "@rollup/plugin-node-resolve": "^9.0.0",
     "d3-request": "^1.0.6",
-    "mocha": "^8.2.0",
+    "mocha": "^9.1.2",
+    "npm-audit-resolver": "^2.3.0",
     "rollup": "^2.32.1",
     "sinon": "^9.2.0",
     "tempy": "^1.0.0"

package/server/columns.js

@@ -6,17 +6,41 @@
 
 Object.defineProperty(exports, '__esModule', { value: true });
 
-function parseColumn(col_spec) {
+const MAX_INTEGER = Math.pow(2, 31) - 1;
+const MAX_RANGE_LENGTH = Math.pow(2, 15);
+
+// Attempt to parse col_spec as a columns spec;
+// return true if we succeed, and false if not.
+function looksLikeMultipleColumns(col_spec) {
+	try {
+		parseColumns(col_spec);
+	}
+	catch (e) {
+		return false;
+	}
+	return true;
+}
+
+function parseColumn(col_spec, is_optional) {
 	col_spec = col_spec.toUpperCase();
 	if (!/^[A-Z]+$/.test(col_spec)) {
-		throw new Error("Invalid column spec: " + col_spec);
+		if (!col_spec) {
+			if (is_optional) return -1; // Use -1 for unassigned optional binding
+			else throw new Error("Non-optional data binding must specify column");
+		}
+		if (looksLikeMultipleColumns(col_spec)) {
+			throw new Error("You can only select one column");
+		}
+		else throw new Error("Invalid column specification: " + col_spec);
 	}
 
 	var col_ix = 0;
 	for (var i = 0; i < col_spec.length; i++) {
 		col_ix = col_ix * 26 + (col_spec.charCodeAt(i) - 64);
 	}
-	return col_ix - 1;
+
+	if (col_ix - 1 > MAX_INTEGER) console.warn("Column index out of range");
+	return Math.min(col_ix - 1, MAX_INTEGER);
 }
 
 function printColumn(col_ix) {
@@ -51,6 +75,11 @@ function parseRange(col_range) {
 	var incrementer = last_ix >= first_ix ? 1 : -1;
 	var n = Math.abs(last_ix - first_ix) + 1;
 
+	if (n > MAX_RANGE_LENGTH) {
+		console.warn("Truncating excessively long range");
+		n = MAX_RANGE_LENGTH;
+	}
+
 	for (var i = 0; i < n; i++) {
 		r.push(first_ix + i*incrementer);
 	}
@@ -83,18 +112,36 @@ function parseColumns(cols_spec) {
 }
 
 function splitIntoRanges(indexes) {
+	if (!indexes.length) {
+		return [];
+	}
 	var ranges = [];
-	var start, end;
-	for (var i = 0; i < indexes.length; i++) {
-		if (i > 0 && indexes[i] == indexes[i-1] + 1) {
-			end = indexes[i];
+	var start = indexes[0], end = indexes[0];
+	var direction = null;
+	for (var i = 0; i < indexes.length - 1; i++) {
+		var diff = indexes[i + 1] - indexes[i];
+		if (direction === null && Math.abs(diff) === 1) {
+			// It's a range with either ascending columns (direction=1), or descending
+			// columns (direction=-1)
+			end = indexes[i + 1];
+			direction = diff;
+			continue;
 		}
-		else {
-			if (typeof start != "undefined") ranges.push([start, end]);
-			start = end = indexes[i];
+
+		if (diff === direction) {
+			// The range is continuing in the same direction as before
+			end = indexes[i + 1];
+			continue;
 		}
+
+		// There's nothing more in the range, so add it, and start a new range from the
+		// next column
+		ranges.push([start, end]);
+		start = end = indexes[i + 1];
+		direction = null;
 	}
-	if (typeof start != "undefined") ranges.push([start, end]);
+	// There will always be a range which hasn't been added at the end
+	ranges.push([start, end]);
 	return ranges;
 }
 
@@ -124,7 +171,7 @@ function parseDataBinding(d, data_table_ids) {
 	r.data_table_id = data_table_ids[data_table_name];
 
 	var col_spec = d[d.type].substr(double_colon_ix + 2);
-	if (d.type == "column") r.column = parseColumn(col_spec);
+	if (d.type == "column") r.column = parseColumn(col_spec, d.optional);
 	else if (d.type == "columns") r.columns = parseColumns(col_spec);
 	else throw new Error("Unknown data binding type: " + d.type);
 

package/server/comms_js.js

@@ -131,8 +131,6 @@ if (template && template.update && template.update.length != 0) {
 }
 `;
 
-module.exports = {
-	withOriginCheck: BEFORE + CHECK_ORIGIN + AFTER,
-	withoutOriginCheck: BEFORE + AFTER,
-	validate: VALIDATE,
-};
+exports.withOriginCheck = BEFORE + CHECK_ORIGIN + AFTER;
+exports.withoutOriginCheck = BEFORE + AFTER;
+exports.validate = VALIDATE;