jsgui3-server 0.0.82 → 0.0.85

package/README.md

@@ -1,3 +1,10 @@
 # Jsgui3 Server
 
-This module covers functionality that enables the application to be served to clients.
+This module covers functionality that enables the application to be served to clients.
+
+2022: Seems as though the server could be bundled with a few useful / essential compilers.
+
+// Just running the Server without a website set up could be a way to test the compilers and data transformers.
+// Would then be able to publish access to the compilers using the server mechanisms.
+
+

package/bundler/bundle.js

@@ -0,0 +1,11 @@
+// Could make this a Collection?
+
+const {Collection} = require('jsgui3-html');
+
+class Bundle extends Collection {
+    constructor(spec = {}) {
+        super(spec);
+    }
+}
+
+module.exports = Bundle;

package/bundler/bundler.js

@@ -0,0 +1,9 @@
+const {Evented_Class} = require('jsgui3-html');
+
+class Bundler extends Evented_Class {
+    constructor(spec = {}) {
+        super(spec);
+    }
+}
+
+module.exports = Bundler;

package/bundler/css-bundler.js

@@ -0,0 +1,56 @@
+const Bundler = require('./bundler');
+const Bundle = require('./bundle');
+const {obs, prom_or_cb} = require('fnl');
+const {tof} = require('jsgui3-html');
+const fnlfs = require('fnlfs');
+const browserify = require('browserify');
+const babel = require('@babel/core');
+const stream_to_array = require('stream-to-array');
+const util = require('util');
+const Stream = require('stream');
+// Will put the JS together. Maybe images?
+
+// Will put the JS together. Maybe images?
+//  Get everything ready to serve.
+
+// Would need a JS file that contains refs to all of the components used.
+//  Examine what is in the website and what JS it needs.
+
+// Should be customisable which system gets used to make the bundle.
+//  eg babel or esbuild. Browserify still seems to work on code here at least, but esbuild seems better.
+
+// JS bundling will become a bit more advanced in this server. Similar principles.
+
+// JS_Bundler reporting css that gets found while bundling JS would make sense.
+
+// jsgui3-jsbuilder could be a separate project too.
+// or jsgui3-js-builder even.
+
+
+// bundle_css_from_js_file?
+// scan_js_file_for_css perhaps.
+// scan_js_for_css
+//  maybe scan an AST or AST stream as it's coming in.
+
+
+
+const bundle_css_from_js = (js_file_path, options = {}) => {
+
+    return obs(async (next, complete, error) => {
+
+        // Go through each file? Just the first?
+        
+    });
+
+}
+
+
+
+class CSS_Bundler extends Bundler {
+    constructor(spec = {}) {
+        super(spec);
+    }
+}
+
+CSS_Bundler.bundle_css_from_js = bundle_css_from_js;
+module.exports = CSS_Bundler;

package/bundler/js-bundler.js

@@ -0,0 +1,214 @@
+const Bundler = require('./bundler');
+const Bundle = require('./bundle');
+const {obs, prom_or_cb} = require('fnl');
+const {tof} = require('jsgui3-html');
+const fnlfs = require('fnlfs');
+const browserify = require('browserify');
+const babel = require('@babel/core');
+const stream_to_array = require('stream-to-array');
+const util = require('util');
+const Stream = require('stream');
+// Will put the JS together. Maybe images?
+
+// Will put the JS together. Maybe images?
+//  Get everything ready to serve.
+
+// Would need a JS file that contains refs to all of the components used.
+//  Examine what is in the website and what JS it needs.
+
+// Should be customisable which system gets used to make the bundle.
+//  eg babel or esbuild. Browserify still seems to work on code here at least, but esbuild seems better.
+
+// JS bundling will become a bit more advanced in this server. Similar principles.
+
+// JS_Bundler reporting css that gets found while bundling JS would make sense.
+
+// jsgui3-jsbuilder could be a separate project too.
+// or jsgui3-js-builder even.
+
+
+
+
+
+
+const bundle_js = (js_file_path, options = {}, callback) => {
+
+    // Could even split a file into its server and client components.
+    // Maybe providing found css would work well here.
+
+
+
+    // Returning an observable and not using a callback would work best.
+
+    const res = obs((next, complete, error) => {
+
+
+
+        let a = arguments;
+        if (typeof a[2] === 'function') {
+            callback = a[2];
+
+            options.include_sourcemaps = true;
+        }
+
+        (async () => {
+            // options
+            // may want a replacement within the client-side code.
+            // Can we call browserify on the code string?
+            //  Creating a modified copy of the file would do.
+            //  Load the file, modify it, save it under a different name
+
+            let s = new Stream.Readable(),
+                path = require('path').parse(js_file_path);
+
+            let fileContents = await fnlfs.load(js_file_path);
+
+
+            // Maybe do that post browserify - can read through whole thing to find the css template literals.
+
+            // Could use the CSS bundler to scan_js_for_css
+            //  Seems as though it would be best as an observable.
+
+
+            // Could first get own system AST of the JS file.
+
+            // Modify the original file contents so that only client-side parts appear?
+            //  Could be done by programatically removing a whole code block, what to do if it is run on the server.
+
+
+            //console.log('1) fileContents.length', fileContents.length);
+            // are there any replacements to do?
+            // options.replacements
+
+            if (options.js_mode === 'debug') {
+                options.include_sourcemaps = true;
+            }
+            if (options.js_mode === 'compress' || options.js_mode === 'mini') {
+                options.include_sourcemaps = false;
+                options.babel = 'mini';
+            }
+
+            //console.log('options.babel', options.babel);
+
+            if (options.replace) {
+                let s_file_contents = fileContents.toString();
+                //console.log('s_file_contents', s_file_contents);
+                each(options.replace, (text, key) => {
+                    //console.log('key', key);
+                    //console.log('text', text);
+                    let running_fn = '(' + text + ')();'
+                    //console.log('running_fn', running_fn);
+                    s_file_contents = s_file_contents.split(key).join(running_fn);
+                })
+                fileContents = Buffer.from(s_file_contents);
+                //console.log('2) fileContents.length', fileContents.length);
+            }
+            // Then we can replace some of the file contents with specific content given when we tall it to serve that file.
+            //  We have a space for client-side activation.
+            s.push(fileContents);
+            s.push(null);
+
+            //let include_sourcemaps = true;
+
+            let b = browserify(s, {
+                basedir: path.dir,
+                //builtins: false,
+                builtins: ['buffer', 'process'],
+                'debug': options.include_sourcemaps
+            });
+
+            // May be able to better put the bundle stream info into the observable results.
+            //  Browserify gets given the stream.
+
+            // Nice if this function could output a stream as well.
+
+
+            let parts = await stream_to_array(b.bundle());
+
+            const buffers = parts
+                .map(part => util.isBuffer(part) ? part : Buffer.from(part));
+            let buf_js = Buffer.concat(buffers);
+            let str_js = buf_js.toString();
+
+            // full browserified (client) js app.
+
+            let babel_option = options.babel
+            //console.log('babel_option', babel_option);
+            if (babel_option === 'es5') {
+
+                let o_transform = {
+                    "presets": [
+                        "es2015",
+                        "es2017"
+                    ],
+                    "plugins": [
+                        "transform-runtime"
+                    ] //,
+                    //'sourceMaps': 'inline'
+                };
+
+                if (options.include_sourcemaps) o_transform.sourceMaps = 'inline';
+                let res_transform = babel.transform(str_js, o_transform);
+                //console.log('res_transform', res_transform);
+                //console.log('Object.keys(res_transform)', Object.keys(res_transform));
+                let jst_es5 = res_transform.code;
+                //let {jst_es5, map, ast} = babel.transform(str_js);
+                //console.log('jst_es5.length', jst_es5.length);
+                buf_js = Buffer.from(jst_es5);
+            } else if (babel_option === 'mini') {
+                /*
+                let o_transform = {
+                    presets: ["minify"]//,
+                    //'sourceMaps': 'inline'
+                };
+                */
+                let o_transform = {
+                    "presets": [
+                        ["minify", {
+                            //"mangle": {
+                            //"exclude": ["MyCustomError"]
+                            //},
+                            //"unsafe": {
+                            //	"typeConstructors": false
+                            //},
+                            //"keepFnName": true
+                        }]
+                    ],
+                    //plugins: ["minify-dead-code-elimination"]
+                };
+                if (options.include_sourcemaps) o_transform.sourceMaps = 'inline';
+
+                let res_transform = babel.transform(str_js, o_transform);
+                buf_js = Buffer.from(res_transform.code);
+            } else {
+                buf_js = Buffer.from(str_js);
+            }
+            complete(buf_js);
+        })();
+
+    })
+    if (callback) {
+        res.on('complete', (value) => {
+            callback(null, value);
+        });
+        res.on('error', err => {
+            callback(err);
+        })
+    } else {
+        return res;
+    }
+    // An observable would be better as result could include status messages (with their timings).
+
+    //return prom_or_cb((resolve, reject) => {
+        
+    //}, callback);
+}
+
+class JS_Bundler extends Bundler {
+    constructor(spec = {}) {
+        super(spec);
+    }
+}
+
+JS_Bundler.bundle_js = bundle_js;
+module.exports = JS_Bundler;

package/bundler/test_ast.js

@@ -0,0 +1,74 @@
+
+// Load in a JS file, such as square box client.
+// Look for the CSS.
+
+
+// Load a JS file into an OO structure
+// Short term goals:
+
+// Answering questions on the level of the file.
+// Questions that are useful for finding out how to link the js files together.
+//  Don't want each file to be given it's own scope - want them to share local variable references to the the necessary things.
+//   Then those variable names won't be reused in inner scopes unless their replacement there is fine (ie they are otherwise unused).
+
+// This will assemble somewhat detailed information about what happens inside a JavaScript file.
+//  The aim is to carry out and represent different kinds of analysis, but at the moment focusing of finding features.
+
+// Root features
+
+// Declaration features
+// Object features
+
+// Let's make it so that any node can be tagged as having / representing / being part of a feature of some kind.
+//  To begin with focus on what variables are being declared and used.
+
+// Will be great to load in a whole load of JSGUI projects in such a way that the system / platform understands the ordering and 
+// recomposes them into a flat system where many / all declarations are local rather than using any import
+
+// jsgui-lang
+// ----------
+
+// Recognise the basic / general signature of the full document.
+// Recognise that the declarations are all within the root
+// Recognise the exports object at the end exports an object that is composed from same-name references to items declared within lang-mini
+//  and only items declared within jsgui
+//  that it has no external references
+
+// JSGUI_JS_File?
+
+
+
+const JS_File = require('../resources/jsbuilder/JS_File/JS_File');
+//const JS_File_Comprehension = require('../JS_File_Comprehension');
+const path = require('path');
+const fs = require('fs');
+const {each} = require('lang-mini');
+
+const JS_AST_Node = require('../resources/jsbuilder/JS_AST_Node_Extended/JS_AST_Node_Extended');
+
+const test_js_file = () => {
+    const fnl_path = require.resolve('./../examples/square_box_client.js');
+    const file_path = fnl_path;
+    // path of lang mini...
+
+    // Write and test a simple and convenient way for analysing JS files and recompiling them.
+    // To start with, find the ways to arrange parts of the JS into 'platforms'.
+    // How to then build platforms into JS files.
+    //  Will be about closures and sequences.
+    //  A lot about unique naming, closures, and getting the sequence of definitions correct.
+    // ObjectPattern
+    const resolved_path = path.resolve(file_path);
+    //console.log('resolved_path', resolved_path);
+
+    const fstream = fs.createReadStream(resolved_path);
+
+    const jsf = JS_File.load_from_stream(fstream, file_path);
+    jsf.on('ready', () => {
+        const {js_ast_node_file} = jsf;
+        //const body_child_node_identifier_names = [];
+        //const map_bcnidns = {};
+    });
+
+}
+
+test_js_file();

package/bundler/webpage-bundler.js

@@ -0,0 +1,286 @@
+const Bundler = require('./bundler');
+
+const JS_Bundler = require('./js-bundler');
+
+const Bundle = require('./bundle');
+const {obs, prom_or_cb} = require('fnl');
+const fnlfs = require('fnlfs');
+const {tof, HTML_Document, Client_HTML_Document, Control} = require('jsgui3-html');
+const util = require('util');
+const Server_Page_Context = require('./../page-context');
+// Will put the JS together. Maybe images?
+//  Get everything ready to serve.
+const browserify = require('browserify');
+const babel = require('@babel/core');
+const stream_to_array = require('stream-to-array');
+
+// Maybe some web pages should be unbundlable? Or we note that they are dynamic (somehow).
+//  Perhaps another fn should do that check. Don't assume all pages will bundle OK. Could raise obs error if needed.
+
+// Bundling CSS from a JS page.
+
+// const bundle_js_to_css
+
+
+// Maybe best to move to a JS Bundler.
+
+// CSS bundler too
+//  Could consult JS source files. Could use the JS bundler to get css from JS.
+
+// Creating working bundles from JS (and other) source files seems like an important task.
+
+const {bundle_js} = JS_Bundler;
+
+
+const bundle_web_page = (webpage, options = {}) => {
+    const {content} = webpage;
+
+    let {disk_path_client_js} = options;
+
+
+    // Could use the CSS bundler to extract the css from the JS.
+
+    
+
+    // Could look for the css within disk_path_client_js.
+
+
+    //if (options.js_client) js_client_disk_path = 
+
+    // Then depending on the content type
+    
+
+    const t_content = tof(content);
+
+    //console.log('content', content);
+    //console.log('t_content', t_content);
+
+    return obs((next, complete, error) => {
+        const res = new Bundle();
+
+        // The observable could / should return updates along the way, things that contribute to the full result.
+
+        if (t_content === 'string') {
+            // Hardly anything to bundle. No JS required, so it seems.
+            // Maybe put it inside a basic JSGUI page control...?
+
+            // Page may still have a title.
+            const html = `<html><head><title>${webpage.title}</title></head><body>${content}</body></html>`;
+            const buff = Buffer.from(html, "utf-8");
+
+            // and value with different types of compression....
+            //  worth having them ready.
+            res.push({
+                'path': webpage.path,
+                'value': buff,
+                'content-type': 'text/html'
+            });
+            complete(res);
+
+        } else if (t_content === 'control') {
+            //console.log ('content.context', content.context);
+            //console.log('content', content);
+
+            // May need to clone this control, putting it into new contexts.
+            //  Or render it with a temporary context?
+
+            // Controls with temportary contexts could be useful.
+            if (content instanceof Control) {
+                //console.log('content is control');
+                //console.log('content.constructor.name', content.constructor.name);
+                if (content instanceof HTML_Document) {
+                    console.log('content is an html document');
+
+                    throw 'NYI';
+                } else {
+
+                    // create an HTML document
+                    //  then put this control (or a clone of it) inside that HTML document.
+
+                    // We may be changing the construction / rendering order here.
+                    //  Seem to be doing more construction of controls without a Page_Context.
+                    //   That page_context may only be important at a later stage.
+
+                    // Some pages will render differently depending on when they are rendered.
+                    //  That may be why a server would need to individually build each page.
+                    //  Eg if there is SSR of latest news items.
+
+                    // Question of pre-rendering or rendering on each page request.
+                    //  Maybe we don't bundle it if it includes dynamic content.
+                    //   Though we would need to bundle the JS.
+
+                    // Could create a new page context.
+
+                    //const cloned_content = content.clone();
+
+                    // Should be able to clone a control?
+
+                    //console.log('cloned_content', cloned_content);
+
+                    // The page context may not have a request and response.
+                    //  For bundling, won't have one (unless one were made).
+
+                    // Creating the JS bundle...
+
+                    // Make it require jsgui-html for the moment.
+                    //  Requiring specific JS of the client control may make more sense....
+
+                    // First let's bundle the JS.
+                    //  Or get the JS bundle for 'jsgui3-html'.
+
+                    // js.serve_package('/js/app.js', js_client, o_serve_package, (err, served) => {
+                    //  maybe should make a JS_Bundler class.
+
+                    // May need to be told earlier what file path we are using for the client js bundle.
+
+                    const diskpath_js_client = disk_path_client_js || require.resolve('jsgui3-html');
+                    //const diskpath_js_client = require.resolve('./../controls/page/admin.js');
+
+                    // Bundle js could be an observable
+                    //  So when it finds CSS, it can output that.
+                    //  A Control's CSS property, within the JS definition.
+                    //   That can be output to a CSS file, copied or removed from the JS file.
+                    bundle_js(diskpath_js_client, {
+                        'js_mode': 'mini',
+                        'babel': 'mini'
+                    }, (err, res_bundle_js) => {
+                        if (err) {
+                            console.trace();
+                            throw err;
+                        } else {
+                            //console.log('res_bundle_js', res_bundle_js);
+
+                            // Could search at this point for class.css properties.
+                            res.push({
+                                'path': webpage.path + 'js/app.js',
+                                'value': res_bundle_js,
+                                'content-type': 'text/javascript'
+                            });
+
+                            const spc = new Server_Page_Context({
+                                //request: 
+                            });
+        
+        
+                            const doc = new Client_HTML_Document({
+                                context: spc
+                            });
+                            doc.include_js('/js/app.js');
+                            doc.body.add(content);
+        
+                            // Getting it to load the JS would be nice...
+                            //  But maybe only do it automatically if it's an active document.
+                            //  If it's just generated JS then best not to bundle JS to load.
+        
+                            // Could also make smaller bundles for some specific parts of the app. Could improve speed.
+                            //console.log('doc.html', doc.html);
+                            const buff_html = Buffer.from(doc.html, "utf-8");
+                            // But add the client-side stuff to that doc.
+        
+                            // Remember, this part is about bundling rather than serving.
+        
+                            res.push({
+                                'path': webpage.path,
+                                'value': buff_html,
+                                'content-type': 'text/html'
+                            });
+                            //console.log('pre complete bundlejs');
+                            //console.log('res.length()', res.length());
+                            complete(res);
+                        }
+                    });
+
+                    // Need to create a new control
+
+                    //throw 'NYI';
+                }
+            } else {
+                throw 'NYI';
+            }
+        } else {
+
+            //console.log('t_content', t_content);
+
+            if (t_content === 'function') {
+
+                const spc = new Server_Page_Context({
+                    //request: 
+                });
+
+                const Ctrl = content;
+                const ctrl = new Ctrl({
+                    'context': spc
+                });
+
+                if (ctrl instanceof HTML_Document) {
+                    console.trace();
+                    throw 'NYI';
+                } else {
+
+
+                    const diskpath_js_client = disk_path_client_js || require.resolve('jsgui3-html');
+                    //const diskpath_js_client = require.resolve('./../controls/page/admin.js');
+
+                    bundle_js(diskpath_js_client, {
+                        'js_mode': 'mini',
+                        'babel': 'mini'
+                    }, (err, res_bundle_js) => {
+                        if (err) {
+                            console.trace();
+                            throw err;
+                        } else {
+                            //console.log('res_bundle_js', res_bundle_js);
+
+                            res.push({
+                                'path': webpage.path + 'js/app.js',
+                                'value': res_bundle_js,
+                                'content-type': 'text/javascript'
+                            });
+
+                            const doc = new Client_HTML_Document({
+                                context: spc
+                            });
+                            doc.include_js('/js/app.js');
+                            doc.body.add(ctrl);
+        
+                            // Getting it to load the JS would be nice...
+                            //  But maybe only do it automatically if it's an active document.
+                            //  If it's just generated JS then best not to bundle JS to load.
+        
+                            // Could also make smaller bundles for some specific parts of the app. Could improve speed.
+                            //console.log('doc.html', doc.html);
+                            const buff_html = Buffer.from(doc.html, "utf-8");
+        
+                            res.push({
+                                'path': webpage.path,
+                                'value': buff_html,
+                                'content-type': 'text/html'
+                            });
+                            //console.log('pre complete bundlejs');
+                            //console.log('res.length()', res.length());
+                            complete(res);
+                        }
+                    });
+                }
+
+            } else if (false) {
+                console.trace();
+                throw 'NYI';
+            } else {
+                console.trace();
+                throw 'NYI';
+            }
+        }
+    });
+    //throw 'NYI';
+}
+
+class Webpage_Bundler extends Bundler {
+    constructor(spec = {}) {
+        super(spec);
+    }
+}
+
+Webpage_Bundler.bundle_web_page = bundle_web_page;
+
+module.exports = Webpage_Bundler;

package/bundler/website-bundler.js

@@ -0,0 +1,23 @@
+const Bundler = require('./bundler');
+const Bundle = require('./bundle');
+const {obs} = require('fnl');
+const {tof} = require('jsgui3-html');
+
+//  Get everything ready to serve.
+
+// Would need a JS file that contains refs to all of the components used.
+//  Examine what is in the website and what JS it needs.
+
+// Bundling up the contents of a small/medium website so that contents are compressed and ready to serve.
+// Could be useful for deploying to Android / iOS devices, maybe as apps.
+
+
+
+class Website_Bundler extends Bundler {
+    constructor(spec = {}) {
+        super(spec);
+    }
+}
+
+
+module.exports = Website_Bundler;