@nebularstreams/statify 3.0.0

package/src/publisher-lib.js

@@ -0,0 +1,179 @@
+/* eslint-disable import/no-dynamic-require */
+const
+	m = require("mithril"),
+	render = require("mithril-node-render"),
+	{Cdn} = require("./util/cdn/nobj.cdn"),
+
+	PageWidget = require("./widgets/Page"),
+	footer = require("./widgets/Cookies"),
+	{convertMarkdownFile} = require("./util/markdown/markdown"),
+	{parseImageUrl} = require("./util/htmlroute"),
+	{getSnippet} = require("./widgets/Snippets"),
+
+	// use lambda to insert .html into the request
+	// but use non-html here
+
+	EXTENSION_HTML = "";
+
+function saniLink(link, newext) {
+	if (link.indexOf(".json")) {
+		return link.split(".json")
+			.join(newext);
+	}
+	return link;
+
+}
+
+function getLevels(basePath, metadata) {
+	return basePath.split("/")
+		.filter((o) => !!o)
+		.reduce((tot, item, idx, all) => {
+			tot.push({name: item, link: (idx ? all[idx - 1] + "/" : "") + item});
+			return tot;
+		}, [])
+		.filter((item) => ((metadata["links.metadata"] || {})[item.link]))
+		.map((item) => ({...item, submeta: metadata["links.metadata"][item.link]}));
+}
+
+function generateLinks(levels, metadata, nextLevelLinks = []) {
+
+	const
+		lastLevel = levels.length ? levels[levels.length - 1] : undefined,
+		previousLevel = levels.length > 1 ? levels[levels.length - 2] : undefined;
+
+	return {
+		...previousLevel
+			? {
+				[previousLevel.name]: {
+					class: "inter",
+					url: [metadata.canonical, previousLevel.link, "index"].join("/"),
+					...(previousLevel.submeta || {})
+				}
+			}
+			: {},
+		...lastLevel
+			? {
+				[lastLevel.name]: {
+					class: "inter",
+					url: [metadata.canonical, lastLevel.link, "index"].join("/"),
+					...(lastLevel.submeta || {})
+				}
+			}
+			: {
+				...(metadata["links.doc"] || {})
+			},
+		...nextLevelLinks
+	};
+}
+
+function getPageMetadata(filename, metadata, relativeRoute, levels, baseFilename) {
+
+	if (filename.endsWith(".md")) {
+
+		const
+
+			relativeFolder = relativeRoute.substring(0, relativeRoute.lastIndexOf("/")),
+			document = convertMarkdownFile(filename, {
+				metadata,
+				relativeRoute: relativeFolder,
+				relativeFile: baseFilename
+			}),
+
+			{images, index} = document,
+			parsedImages = images.map(
+				(image) => parseImageUrl(image, metadata, relativeFolder)
+			),
+			lastLevel = levels.length ? levels[levels.length - 1] : undefined;
+
+		return {
+			title: index[0].text,
+			description: index[0].next,
+			submeta: lastLevel && lastLevel.submeta,
+			widget: "Markdown",
+			image: {
+				url: images[0]
+			},
+			document,
+			links: generateLinks(
+				levels,
+				metadata,
+				{
+					[index[0].text]: {
+						class: "grows"
+					}
+				}
+			),
+			snippet: getSnippet(
+				"Article",
+				index[0].text,
+				parsedImages,
+				undefined,
+				metadata.author,
+				metadata.publisher
+			)
+		};
+	}
+	return require(filename);
+
+}
+
+async function publish(filename, source, htmlRoot = "pages", dynamicData = undefined) {
+
+	if (filename && source) {
+
+		const
+			metadata = require(source + "/metadata.json"),
+			{links, navibar, site} = metadata,
+			{name, domain, nocookies, cookies} = site,
+			filePath = filename.split("/"),
+			navibarLinks = navibar?.links || links;
+
+		filePath.shift();
+
+		const
+			relativeRoute = saniLink(filePath.join("/"), EXTENSION_HTML),
+			baseSource = filePath.pop(),
+			basePath = filePath.join("/");
+
+		if (!domain) {
+			throw new Error("Please set domain in metadata");
+		}
+
+		if (!navibarLinks) {
+			throw new Error("Please set navigation bar in metadata, or define missing page");
+		}
+
+		const
+
+			cdnModel = new Cdn(domain),
+			absoluteFileRoute = [source, htmlRoot, ...basePath ? [basePath] : [], baseSource].join("/"),
+			levels = getLevels(basePath, metadata),
+			pageMetadata = getPageMetadata(absoluteFileRoute, metadata, relativeRoute, levels, baseSource),
+
+			cookiesDialog = nocookies ? undefined : (footer[cookies || "full"](name, domain)),
+
+			result = await render(
+				m(PageWidget, {
+					baseSource,
+					basePath,
+					pageMetadata,
+					relativeRoute,
+					levels,
+					metadata,
+					dynamicData,
+					source,
+					cdnModel,
+					...cookiesDialog && {
+						footer: cookiesDialog
+					}
+				})
+			);
+		return ["<!DOCTYPE html>", result].join("\n");
+
+	}
+
+	throw new Error("No file specified");
+
+}
+
+module.exports = publish;

package/src/publisher-list.js

@@ -0,0 +1,91 @@
+/* eslint-disable import/no-dynamic-require,no-await-in-loop */
+const
+	axios = require("axios"),
+	fs = require("fs"),
+	{ tokenizeStrict } = require("./util/tokenizer"),
+	publish = require("./publisher-lib");
+
+function parseArgs(argv) {
+
+	const
+		name = "publisher-list.js",
+		initial = argv[0].endsWith(name)
+			? 1 : argv[1].endsWith(name)
+				? 2
+				: 0;
+	if (initial) return argv.slice(initial, argv.length);
+	return null;
+}
+
+function stderr(...data) {
+	process.stderr.write(data.join(" ") + "\n");
+}
+
+async function parseItem(source, listname, metadata, type, item, outpath) {
+	stderr("+ Found Item, list", listname, "type", type, "item", item.id);
+	return publish("./" + listname, source, "lists", { meta: metadata, item, relative: outpath }, metadata);
+}
+
+async function writeFile(outpath, contents) {
+
+	// const outfile = outpath + "/index.html";
+	const outfile = outpath + ".html";
+
+	stderr("- Generating page", outfile);
+	if (!fs.existsSync(outpath)) {
+		stderr("- Creating folder", outpath);
+		fs.mkdirSync(outpath, { recursive: true });
+	}
+	fs.writeFileSync(outfile, contents, { encoding: "utf-8" });
+}
+
+async function parseList(source, metadata, listname, list, outpublish, filtername) {
+	const { items, type } = list;
+
+	stderr("> Parsing list type", type, "items", items.length, "filter", filtername);
+
+	for (const item of items) {
+
+		if (filtername && item.id !== filtername) continue;
+
+		const
+			outpath = tokenizeStrict(metadata.listout, { item, type }),
+			html = await parseItem(source, listname, metadata, type, item, outpath);
+
+		await writeFile(outpublish + (outpath ? "/" + outpath : ""), html);
+
+	}
+	stderr("< List", listname, "parsed");
+}
+
+async function init() {
+
+	const
+		args = parseArgs(process.argv);
+
+	if (args[0] && args[1] && args[2]) {
+
+		const
+			[listname, source, outpublish, filtername] = args,
+			metadata = require(source + "/lists/" + listname);
+
+		stderr("- Loading List", listname, "from", metadata.list);
+		stderr("- Source folder", source);
+
+		const
+			list = await axios({ url: metadata.list });
+
+		await parseList(source, metadata, listname, list.data, outpublish, filtername);
+	} else {
+		throw new Error("Usage: publisher-list listname source output filterid");
+	}
+}
+
+init()
+	.then(() => {
+		stderr("! Finished.");
+	})
+	.catch((e) => {
+		console.error(e);
+		process.exitCode = 1;
+	});

package/src/publisher.js

@@ -0,0 +1,38 @@
+/* eslint-disable import/no-dynamic-require */
+const
+	publish = require("./publisher-lib");
+
+function parseArgs(argv) {
+
+	const
+		name = "publisher.js",
+		initial = argv[0].endsWith(name)
+			? 1 : argv[1].endsWith(name)
+				? 2
+				: 0;
+	if (initial) return argv.slice(initial, argv.length);
+	return null;
+}
+
+function init() {
+
+	const
+		args = parseArgs(process.argv);
+
+	if (args[0] && args[1]) {
+
+		const
+			[filename, source] = args;
+
+		publish(filename, source)
+			.then((txt) => {
+				console.log(txt);
+			})
+			.catch((e) => {
+				console.error(e);
+				process.exit(1);
+			});
+	}
+}
+
+init();

package/src/util/cdn/nobj.cdn.js

@@ -0,0 +1,88 @@
+function Cdn(ncsdom) {
+
+    const
+        PROTO = "https://",
+        PUBLICCDNROOT = PROTO + ncsdom,
+        PUBLICPLATFORMROOT = PUBLICCDNROOT + "/all",
+        PUBLICRESOURCES = PUBLICCDNROOT + "/res",
+        PUBLICCHANNELROOT = PUBLICCDNROOT + "/cdn",
+        PUBLICLISTROOT = PUBLICPLATFORMROOT + "/list";
+
+    function channelPath(name, path) {
+        return PUBLICCHANNELROOT + "/" + name + (path ? "/" + path : "");
+    }
+
+    function platformPath(path) {
+        return PUBLICPLATFORMROOT + (path ? "/" + path : "");
+    }
+
+    function resourcesPath(path) {
+        return PUBLICRESOURCES + (path ? "/" + path : "");
+    }
+
+    function platformListPath(entityType, name = "index") {
+        return PUBLICLISTROOT + "/" + entityType + (name ? "/list." + name + ".json" : "");
+    }
+
+    function platformFilesPath(entityType, name, path) {
+        return PUBLICPLATFORMROOT + "/files" + (path ? "/" + path : "");
+    }
+
+    // cdn/nebular/world/list.index.json
+    function channelListPath(channelName, entityType, listName = "index") {
+        return channelPath(channelName, entityType + "/list." + listName + ".json");
+    }
+
+    function channelFilesPath(channelName, path) {
+        return channelPath(channelName, "files/" + (path ? "/" + path : ""));
+    }
+
+    function channelEntityPath(channelName, entityType, entityName, path = "index.json", short = false) {
+        return channelPath(channelName, (short ? "" : "obj/") + entityType + "/" + entityName + (path ? "/" + path : ""));
+    }
+
+    return {
+        channel: {
+            // @deprecated
+            listPath: channelListPath,
+            entityPath: channelEntityPath,
+            filesPath: channelFilesPath,
+            path: channelPath,
+            list: channelListPath,
+        },
+        platform: {
+            path: platformPath,
+            list: platformListPath,
+
+            // @deprecated
+            listPath: platformListPath,
+
+            filesPath: platformFilesPath,
+            resources: resourcesPath,
+            channel: name => new CdnChannel(ncsdom, name)
+        }
+    };
+}
+
+function CdnChannel(ncsdom, channelName) {
+
+    const
+        {channel} = new Cdn(ncsdom),
+        {
+            list, entityPath, path, filesPath
+        } = channel;
+
+    return {
+
+        // @deprecated
+        listPath: (entityType, listName) => list(channelName, entityType, listName),
+
+        entityPath: (entityType, entityName, rest) => entityPath(channelName, entityType, entityName, rest),
+        filesPath: rest => filesPath(channelName, rest),
+        path: rest => path(channelName, rest),
+        list: (entityType, listName) => list(channelName, entityType, listName),
+    };
+
+}
+
+module.exports = {Cdn, CdnChannel};

package/src/util/htmlroute.js

@@ -0,0 +1,39 @@
+const
+	absoRoute = (url, metadata, relativeRoute, relativeFile = "index") => (
+		((url.indexOf("://") !== -1) || url.startsWith("mailto:"))
+			? url
+			: (url.startsWith("#"))
+				? ([
+					metadata.canonical,
+					...relativeRoute
+						? [relativeRoute]
+						: [],
+					relativeFile
+				]).join("/") + url
+				: (url.startsWith("/"))
+					? [metadata.site.url, url.substring(1)].join("/")
+					: ([
+						metadata.canonical,
+						...relativeRoute
+							? [relativeRoute]
+							: [],
+
+						...url
+							? [url]
+							: []
+					].join("/"))
+	);
+const
+	absoFolder = (url, metadata, relativeRoute) => {
+		if (url.indexOf("://") !== -1) return url;
+		if (url.startsWith("/")) return [metadata.canonical, url].join("/");
+		const relati = relativeRoute.split("/");
+		relati.pop();
+		const finalRelate = relati.join("/");
+		return ([metadata.canonical, ...finalRelate ? [finalRelate] : [], url].join("/"));
+	},
+
+	parseImage = ({ url, alt }, metadata, relativeRoute) => ({ url: absoFolder(url, metadata, relativeRoute), alt, "@type": "ImageObject" }),
+	parseImageUrl = (url, metadata, relativeRoute) => (absoFolder(url, metadata, relativeRoute));
+
+module.exports = { absoRoute, absoFolder, parseImage, parseImageUrl };

package/src/util/markdown/lib.showdown.icon.js

@@ -0,0 +1,62 @@
+/**
+ * Showdown Icon Extension, Glyphicon and FontAwesome support for showdown
+ * http://github.com/dbtek/showdown-icon
+ * 2014, Ismail Demirbilek
+ * License: MIT
+ */
+(function (extension) {
+
+	// UML - Universal Module Loader
+	// This enables the extension to be loaded in different environments
+	if (typeof showdown !== "undefined") {
+		// global (browser or nodejs global)
+		extension(showdown);
+	} else if (typeof define === "function" && define.amd) {
+		// AMD
+		define(["showdown"], extension);
+	} else if (typeof exports === "object") {
+		// Node, CommonJS-like
+		module.exports = extension(require("showdown"));
+	} else {
+		// showdown was not found so we throw
+		throw Error("Could not find showdown library");
+	}
+
+}(function (showdown) {
+	"use strict";
+
+	//This is the extension code per se
+
+	// Here you have a safe sandboxed environment where you can use "static code"
+	// that is, code and data that is used accros instances of the extension itself
+	// If you have regexes or some piece of calculation that is immutable
+	// this is the best place to put them.
+
+	// The following method will register the extension with showdown
+	showdown.extension("icon", function () {
+
+		return [
+			{
+				type: "lang",
+				regex: "\\B(\\\\)?@glyphicon-([\\S]+)\\b",
+				replace(a, b, c) {
+					return b === "\\" ? a : `<span class="glyphicon glyphicon-${c}"></span>`;
+				}
+			},
+			{
+				type: "lang",
+				regex: "\\B(\\\\)?@fa-([\\S]+)\\b",
+				replace(a, b, c) {
+					return b === "\\" ? a : `<i class="fa fa-${c}"></i>`;
+				}
+			},
+			{
+				type: "lang",
+				regex: "\\B(\\\\)?@fabrd-([\\S]+)\\b",
+				replace(a, b, c) {
+					return b === "\\" ? a : `<i class="fa fabrd fa-${c}"></i>`;
+				}
+			}
+		];
+	});
+}));

package/src/util/markdown/lib.showdown.yt.js

@@ -0,0 +1,54 @@
+/**
+ * Showdown Icon Extension, Glyphicon and FontAwesome support for showdown
+ * http://github.com/dbtek/showdown-icon
+ * 2014, Ismail Demirbilek
+ * License: MIT
+ */
+(function (extension) {
+	"use strict";
+
+	// UML - Universal Module Loader
+	// This enables the extension to be loaded in different environments
+	if (typeof showdown !== "undefined") {
+		// global (browser or nodejs global)
+		extension(showdown);
+	} else if (typeof define === "function" && define.amd) {
+		// AMD
+		define(["showdown"], extension);
+	} else if (typeof exports === "object") {
+		// Node, CommonJS-like
+		module.exports = extension(require("showdown"));
+	} else {
+		// showdown was not found so we throw
+		throw Error("Could not find showdown library");
+	}
+
+}(function (showdown) {
+	"use strict";
+
+	function getYouTubeImage(ref) {
+		return `https://i3.ytimg.com/vi/${ref}/hqdefault.jpg`;
+	}
+
+	//This is the extension code per se
+
+	// Here you have a safe sandboxed environment where you can use "static code"
+	// that is, code and data that is used accros instances of the extension itself
+	// If you have regexes or some piece of calculation that is immutable
+	// this is the best place to put them.
+
+	function getVideo(id) {
+		return "<md-wrap><md-yt><iframe data-img=\"" + getYouTubeImage(id) + "\" data-src=\"https://www.youtube.com/embed/" + id + "?rel=0&modestbranding=1\" allow=\"encrypted-media; picture-in-picture\" allowfullscreen></iframe></md-yt></md-wrap>";
+	}
+
+	// The following method will register the extension with showdown
+	showdown.extension("youtube", function () {
+		return [{
+			type: "lang",
+			regex: "\\B(\\\\)?@yt-([a-zA-Z0-9_\\-]+)\\b",
+			replace: function (a, b, c) {
+				return b === "\\" ? a : getVideo(c);
+			}
+		}];
+	});
+}));

package/src/util/markdown/markdown.js

@@ -0,0 +1,144 @@
+/* eslint-disable import/no-dynamic-require */
+const
+	fs = require("fs"),
+	showdown = require("showdown"),
+	{ absoRoute } = require("../htmlroute"),
+	MARKDOWN_EXTENSION = ".md"; // ".html"
+
+require("./lib.showdown.icon");
+require("./lib.showdown.yt");
+
+function getHrefLink(link, options) {
+	return absoRoute(link, options.metadata, options.relativeRoute, options.relativeFile)
+		.split(MARKDOWN_EXTENSION)
+		.join("");
+}
+
+function getImageLink(link, options) {
+	return absoRoute(link, options.metadata, options.relativeRoute);
+}
+
+function replaceLink(html, pattern, getLink, options) {
+
+	// noinspection JSUnresolvedVariable,JSUnresolvedFunction
+	const
+		parts = html.split(pattern)
+			.map((linkPart, index) => {
+
+				if (!index) {
+					return linkPart;
+				}
+
+				const
+					linkEndIndex = linkPart.indexOf("\"");
+
+				if (linkEndIndex === -1 || !getLink) {
+					return linkPart;
+				}
+
+				const
+					link = linkPart.substring(0, linkEndIndex),
+					remaining = linkPart.substring(linkEndIndex);
+
+				return getLink(link, options) + remaining;
+
+			});
+
+	return parts.join(pattern);
+
+}
+
+function convert(text, options = {}) {
+
+	// noinspection JSUnresolvedVariable,JSUnresolvedFunction
+	const
+		Converter = new showdown.Converter({
+			ghCompatibleHeaderId: true,
+			openLinksInNewWindow: false,
+			strikethrough: true,
+			emoji: true,
+			metadata: true,
+			tasklist: true,
+			tables: true,
+			moreStyling: true,
+			extensions: ["icon", "youtube"]
+		}),
+		metadata = Converter.getMetadata(undefined);
+
+	// noinspection JSUnresolvedFunction
+	let
+		html = Converter.makeHtml(text);
+
+	html = replaceLink(html, " href=\"", getHrefLink, options);
+	html = replaceLink(html, " src=\"", getImageLink, options);
+	html = html
+		.split("<table>")
+		.join("<div class=\"w-100\"><div class=\"scrollx\"><table>")
+		.split("</table>")
+		.join("</table></div></div>");
+
+	return { html, metadata };
+
+}
+
+function getId(line) {
+	const idpart = line.split("id=\"")[1];
+	return idpart ? idpart.split("\"")[0] : undefined;
+}
+
+function insertImage(line, index) {
+	const img = line.split("<img src=\"")[1];
+	if (img) {
+		index.push(img.split("\"")[0]);
+	}
+}
+
+function insertIndex(line, level, index, indexed, next) {
+	if (line.startsWith("<h" + level)) {
+		const id = getId(line),
+			item = {
+				level,
+				id,
+				text:
+					(line.substring(line.indexOf(">") + 1))
+						.split("</h" + level + ">")[0],
+				next: next.substring(next.indexOf(">") + 1)
+					.split("<")[0]
+			};
+		index.push(item);
+		indexed[id] = item;
+	}
+}
+
+function convertMarkdown(text, options = {}) {
+
+	const
+		output = convert(text, options),
+		index = [],
+		images = [],
+		indexed = {},
+		buffer = output.html.split("\n");
+
+	let i = 0;
+	for (const line of buffer) {
+		insertIndex(line, 1, index, indexed, buffer[i + 1]);
+		insertIndex(line, 2, index, indexed, buffer[i + 1]);
+		insertIndex(line, 3, index, indexed, buffer[i + 1]);
+		insertImage(line, images);
+		i++;
+	}
+
+	return { html: output.html, index, indexed, images };
+
+}
+
+function convertMarkdownFile(path, options = {}) {
+
+	const
+		text = fs.readFileSync(path, { encoding: "UTF-8" });
+
+	return convertMarkdown(text, options);
+
+}
+
+module.exports = { convert, convertMarkdown, convertMarkdownFile };