@naanlang/naan 1.0.9 → 1.0.11

package/lib/env_nodeworker.js

@@ -137,10 +137,15 @@ exports.NaanWorkerActivate = function NaanWorkerActivate(cbReady) {
                     naanlib.js.h = naanlib.js.d;                            // set host directory path
                 if (msg.dirpath)
                     naanlib.js.d = msg.dirpath;                             // set owner directory path
+                if (msg.altcmd)
+                    naanlib.textLine(msg.altcmd);
+                /*
+                 ### already started ###
                 naanlib.start({
                     state: msg.state,
                     cmd: msg.altcmd
                 });
+                */
             }
             else if (msg.id == "import")
                 importScripts(msg.script);                                  // load a JavaScript

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@naanlang/naan",
-    "version": "1.0.9",
+    "version": "1.0.11",
     "author": "Richard C. Zulch",
     "description": "Naan™ software platform",
     "main": "./lib/core/naanlib.js",

package/plugins/openSearch/openSearch.nlg

@@ -0,0 +1,490 @@
+/*
+ * openSearch.nlg
+ * openSearch
+ *
+ *     OpenSearch client.
+ *
+ * column positioning:                          //                          //                      !
+ *
+ * Copyright (c) 2023 by Richard C. Zulch
+ *
+ */
+
+/*
+ * openSearchIndex
+ *
+ * Access object for API functions on a single index.
+ *
+ * This client provides a custom datatype facility as a layer on top of OpenSearch. A schema defines
+ * the fields in an index, which are compiled into a mapping dictionary. The extra layer allows for
+ * adding additional configuration to the field.
+ *
+ * The inbuilt schema types are:
+ *      `literal                                -- keyword
+ *      `title                                  -- text for a title
+ *      `hashtags                               -- set of hashtags
+ *      `date                                   -- date
+ *      `boolean                                -- boolean
+ *      `currency                               -- currency with type
+ *      `text                                   -- block of text
+ *      `name                                   -- proper name, first-last-middle-etc.
+ *      `integer                                -- integer
+ *      `url                                    -- URL
+ *
+ * Current options are:
+ *  {
+ *      languages: [
+ *          <language>: {
+ *              keywords: [<strings>...]        -- lowercase keywords not to stem
+ *          }
+ *          ...
+ *      ]
+ *  }
+ *
+ */
+
+closure openSearchIndex(prosc, indexName, ioptions, local osin) {
+    global()
+    osin = new(object, this)
+    if symbol(indexName)
+        indexName = indexName.tostring()
+    osin.analyzers = {                                                      // map names, language to analyzer
+        english: "english"
+    }
+
+    // langKeywords
+    //
+    // Customize the specified language with an array of keywords that will not be stemmed. This is
+    // useful for proper names like Дарья and Таня which otherwise become three-letter words that
+    // match too many things. Note that the keywords must be lowercase.
+    //
+    function langKeywords(language, keywords,
+        local keywordFilter, stopFilter, stemFilter, params) {
+        keywordFilter = "naan_${language}_keywords"
+        stopFilter = "naan_${language}_stop"
+        stemFilter = "naan_${language}_stemmer"
+        filters = [ "lowercase", keywordFilter, stopFilter, stemFilter ]
+        params = {
+            settings: {
+                analysis: {
+                    filter: { }
+                    analyzer: { }
+                }
+            }
+        }
+        osin.analyzers[language] = "naan_${language}"
+        params.settings.analysis.analyzer[osin.analyzers[language]] = {
+            tokenizer: "standard"
+            filter: filters
+        }
+        params.settings.analysis.filter[keywordFilter] = {
+            type: "keyword_marker"
+            keywords: keywords
+        }
+        params.settings.analysis.filter[stopFilter] = {
+            type: "stop"
+            stopwords: "_${language}_"
+        }
+        params.settings.analysis.filter[stemFilter] = {
+            type: "stemmer"
+            language: language
+        }
+        params
+    }
+
+    // textRawMF
+    //
+    // Return a text+raw multifield in the languages configured for our client.
+    //
+    function textRawMF(local fieldspec, language) {
+        fieldspec = {
+            type: "text"
+            fields: {
+                raw: {
+                    "type":         "keyword"
+                }
+            }
+        }
+        for language in osin.analyzers
+            fieldspec.fields[language] = {                                  // add languages
+                type: "text"
+                analyzer: osin.analyzers[language]
+            }
+        fieldspec
+    }
+    
+    // makeMapping
+    //
+    // Make an OpenSearch mapping dictionary for the specified dictionary of field names/types.
+    //
+    function makeMapping(fields, local mapping, fname, ftype, osfield) {
+        osin.fields = fields
+        osin.fieldNames = []
+        mapping = { }
+        for `(fname, ftype) in fields {
+            osin.fieldNames.push(fname)
+            osfield = eval({
+                //
+                // literal
+                //
+                literal: quote({
+                    type: "keyword"
+                })
+                //
+                // title
+                //
+                title: quote(
+                    textRawMF()
+                )
+                //
+                // hashtags
+                //
+                hashtags: quote(
+                    textRawMF()
+                )
+                //
+                // date
+                //
+                date: quote(
+                    textRawMF()
+                )
+                //
+                // boolean
+                //
+                boolean: quote(
+                    textRawMF()
+                )
+                //
+                // currency
+                //
+                currency: quote(
+                    textRawMF()
+                )
+                //
+                // text
+                //
+                text: quote(
+                    textRawMF()
+                )
+                //
+                // name
+                //
+                name: quote(
+                    textRawMF()
+                )
+                //
+                // integer
+                //
+                integer: quote(
+                    textRawMF()
+                )
+                //
+                // url
+                //
+                url: quote(
+                    textRawMF()
+                )
+            }[ftype])
+            if !osfield
+                debuglog("OpenSearchClient.makeMapping: unknown field type ${ftype}")
+            else {
+                osfield.meta = {
+                    type: ftype
+                }
+                mapping[fname] = osfield
+            }
+        }
+        mapping
+    }
+    
+    // bulkDocs
+    //
+    // Create a bulk docs request body with the specified operation. Returns a standard 
+    // `(error, string) result tuple.
+    //
+    function bulkDocs(docs, action, local items, doc, index, operation, error, data) {
+        items = []
+        for `(doc, index) in docs {
+            operation = { }
+            operation[action] = { _index: indexName }
+            `(error, data) = JsonStringify(operation)
+            if !error
+                `(error, doc) = JsonStringify(doc)
+            if error {
+                error = Error("openSearchIndex.write: doc[${index}] unwritable:", error)
+                ErrorDebuglog(error)
+                return (list(error))
+            }
+            items.push(data)
+            items.push(doc)
+        }
+        list(false, items.join("\n").concat("\n"))
+    }
+
+    // schema
+    //
+    // Get the schema for the named index.
+    //
+    osin.schema = closure schema(local error, data, fields, key, osfield) {
+        `(error, data) = prosc.request(indexName.concat("/_mapping"), {
+            method: "GET"
+        })
+        if error
+            return (list(error))
+        osin.fieldNames = []
+        fields = {}
+        for `(key, osfield) in data[indexName].mappings.properties {
+            osin.fieldNames.push(key)
+            fields[key] = osfield.meta.type
+        }
+        list(false, fields)
+    }
+            
+    // createIndex
+    //
+    // Create the index with the specified field types.
+    //
+    osin.createIndex = closure createIndex(fields, local params, language, opts) {
+        params = { }
+        if ioptions.languages {
+            for `(language, opts) in ioptions.languages
+                if opts.keywords
+                    params = merge(params, langKeywords(language, opts.keywords))
+                else
+                    osin.analyzers[language] = language
+        }
+        params = merge(params, {
+            mappings: {
+                properties: makeMapping(fields)
+            }
+        })
+        // printline("OpenSearchClient.create:\n${Dialect.print(params)}")
+        prosc.request(indexName, {
+            method: "PUT"
+            putdata: params
+        })
+    }
+            
+    // deleteIndex
+    //
+    // Delete the index.
+    //
+    osin.deleteIndex = closure deleteIndex() {
+        prosc.request(indexName, {
+            method: "DELETE"
+        })
+    }
+
+    // write
+    //
+    // Write a document or an array of documents to our index. Returns a standard `(error, data) 
+    // tuple where the data is the _id (or [_ids]) when there was no error.
+    //
+    osin.write = closure write(doc, local error, docs, data) {
+        if array(doc) {
+            `(error, docs) = bulkDocs(doc, "create")
+            if docs
+                `(error, data) = prosc.request(indexName.concat("/_bulk"), {
+                    method: "POST"
+                    putdata: docs
+                    contentType: "application/json"
+                })
+            if data
+                data = data.items.map(function(item) {
+                    item.create._id
+                })
+        }
+        else {
+            `(error, data) = write([doc])
+            if data
+                data = data.0
+        }
+        list(error, data)
+    }
+
+    // update
+    //
+    // Update a document in our index.
+    //
+    osin.update = closure update(doc, _id) {
+        prosc.request(indexName.concat("/_update/${_id}"), {
+            method: "POST"
+            putdata: {
+                doc: doc
+            }
+        })
+    }
+
+    // delete
+    //
+    // Remove a document from our index.
+    //
+    osin.delete = closure delete(_id) {
+        prosc.request(indexName.concat("/_doc/${_id}"), {
+            method: "DELETE"
+        })
+    }
+
+    // search
+    //
+    // Simple search on our index, default to all fields. Options are:
+    //  {
+    //      where:  <array>                         -- fields to search, defaulting to * (all)
+    //      qopts:  <dictionary>                    -- OpenSearch query options
+    //  }
+    //
+    osin.search = closure search(what, soptions, local where, params) {
+        where = soptions.where
+        if !array(where)
+            where = ["*"]
+        params = {
+            query: {
+                multi_match: {
+                    query: what
+                    fields: where
+                }
+            }
+        }
+        if dictionary(soptions.qopts)
+            params.query.multi_match = merge(params.query.multi_match, soptions.qopts)
+        // printline("OpenSearchClient.search:\n${Dialect.print(params)}")
+        prosc.request(indexName.concat("/_search"), {
+            method: "POST"
+            putdata: params
+        })
+    }
+
+    // finis
+    
+    osin
+};
+
+
+/*
+ * OpenSearchClient
+ *
+ * An OpenSearchClient client as a wrapper for the REST API. Options are:
+ *  {
+ *      url:    <string>                        -- host URL with port
+ *      cacert: <data>                          -- CA pem for host
+ *      auth:   <user>:<password>               -- basic authentication
+ *      debug:  <boolean>                       -- debuglog additional error information
+ *      languages: [
+ *          <language>: {
+ *              keywords: [<strings>...]        -- lowercase keywords not to stem
+ *          }
+ *          ...
+ *      ]
+ *  }
+ *
+ */
+
+closure OpenSearchClient(goptions, local oscl, prosc) {
+    global(JSpath)
+    oscl = new(object, this)
+    prosc = new(object, this)
+    prosc.oscl = oscl
+    goptions = new(goptions)
+    if goptions.url.slice(-1) != "/"
+        goptions.url = goptions.url.concat("/")
+
+    // request
+    //
+    // Perform a request on the host as specified.
+    //
+    prosc.request = closure request(path, roptions, local url, error, data) {
+        if roptions
+            roptions = new(roptions)
+        else
+            roptions = { }
+        roptions.debug = goptions.debug
+        roptions.auth = goptions.auth
+        roptions.cacert = goptions.cacert
+        roptions.allowSelfSigned = true
+        url = goptions.url.concat(path)
+        `(error, data) = HttpsApiRequest(url, roptions)
+    }
+    
+    // ping
+    //
+    // Ping the cluster and get basic ID and version information.
+    //
+    oscl.ping = closure ping() {
+        request("")
+    }
+    
+    // index
+    //
+    // Return an object for the named index, which may not exist yet.
+    //
+    oscl.index = closure index(indexName) {
+        openSearchIndex(prosc, indexName, goptions)
+    }
+
+    // search
+    //
+    // Simple search on all indexes and fields. Options are:
+    //  {
+    //      indexes:    <array>                     -- indexes to search, default all
+    //      size:       <integer>                   -- results to return, default 10
+    //      query:      <dictionary>                -- OpenSearch query, default multi_match
+    //      qopts:      <dictionary>                -- OpenSearch query options for multi_match
+    //      filter:     <dictionary>                - OpenSearch query filter
+    //  }
+    //
+    oscl.search = closure search(what, soptions, local params, query, path) {
+        params = { }
+        query = {
+            multi_match: {
+                query: what
+                fields: "*"
+                fuzziness: "auto"
+            }
+        }
+        if soptions.size
+            params.size = soptions.size                                     // results to return
+        if soptions.query
+            query = soptions.query
+        if dictionary(soptions.qopts)
+            query.multi_match = merge(query.multi_match, soptions.qopts)
+        if dictionary(soptions.filter) {
+            query = {
+                bool: {
+                    must: query
+                    filter: soptions.filter
+                }
+            }
+        }
+        params.query = query
+        path = "_search"
+        if soptions.indexes
+            path = soptions.indexes.join(",").concat("/", path)
+        prosc.request(path, {
+            method: "POST"
+            putdata: params
+        })
+    }
+
+    // finis
+    
+    oscl
+};
+
+
+/*
+ * opseInit
+ *
+ *     Initialize the module.
+ *
+ */
+
+function opseInit(local manifest) {
+    manifest = `(openSearchIndex, OpenSearchClient, opseInit)
+
+    Naan.module.build(module.id, "openSearch", function(modobj, compobj) {
+        compobj.manifest = manifest
+        require("frameworks/common").LiveImport()
+        letimport(require("frameworks/node/https_request.nlg"))
+        modobj.exports.OpenSearchClient = OpenSearchClient
+    })
+} ();

package/plugins/openSearch/os_dynamo.nlg

@@ -0,0 +1,195 @@
+/*
+ * os_dynamo.nlg
+ * openSearch
+ *
+ *     OpenSearch-DynamoDB integration.
+ *
+ * column positioning:                          //                          //                      !
+ *
+ * Copyright (c) 2023 by Richard C. Zulch
+ *
+ */
+
+/*
+ * OSDynIndexer
+ *
+ * Index a dynamoDB database into an OpenSearch index. The index should be created before this is
+ * called, with the proper fields and types already configured. The database is the source of truth,
+ * so the index can be recreated from scratch if necessary. Source _ids are assigned by OpenSearch
+ * and therefore this creates all new records. It is not a sync tool.
+ *     This operates by scanning the database for a page of records, then starting the indexing
+ * operation in the background, scanning another batch, and then waiting for the previous background 
+ * index to complete. Each time it has both a set of database records and the ids from indexing, it 
+ * can optionally update the database records with the newly-assigned ID. There is never more than
+ * one database or index operation proceeding at the same time, but they overlap.
+ *
+ * Options:
+ *  {
+ *      chunkSize:      <integer>               // size of each chunk of transfer
+ *      dbFilter:       <array-or-dict>         // DynamoDB KeyExpression(s)
+ *      converter:      <procedure>             // convert DB record to index info
+ *      progress:       <procedure>             // called for each batch completed
+ *      writeID:        <string>                // write OpenSearch _id to database with key writeID
+ *  }
+ *
+ * The converter protocol is:
+ *      converter(dbRecord)                     // return _source doc for database record
+ * The progress protocol is:
+ *      progress(completed)                     // integer number of records completed
+ *
+ */
+
+closure OSDynIndexer(index, table, options,
+    local dbOptions, scanPending, indexPending, updatePending, output, error, records, data, ids) {
+    if !options.converter
+        return (Error("OSDynIndexer: options.converter is required"))
+    dbOptions = {
+        filter: options.filter
+        limit: options.chunkSize || 100
+    }
+    //
+    // Scan records in the database, returning a nonce signaled with records array
+    //
+    closure scanRecords(local pending) {
+        if !output
+            output = []                                                     // just starting to scan records
+        else if !dbOptions.paging
+            return (false)                                                  // scanned all records
+        pending = new(nonce)
+        future(function() {
+            pending.signal(table.scanRecords(dbOptions))
+        },0)
+        pending
+    }
+    //
+    // Update ids into the database, returning a nonce signaled with error array.
+    //
+    closure updateRecordIDs(docs, ids, writeID) {
+        asyncArray(docs, 10, function(entry, index, cancel, local error) {
+            ignore(error)
+            `(error) = table.updateRecord(
+                entry[table.hashKey],
+                entry[table.rangeKey], {
+                set: [list(writeID, ids[index])]
+            })
+            error
+        })
+    }
+    //
+    // Batch index documents into OpenSearch, returning a nonce signaled with ids array.
+    //
+    closure batchIndex(records, local pending, docs) {
+        pending = new(nonce)
+        docs = records.map(function(record) {
+            options.converter(record)
+        })
+        future(function(local error, ids) {
+            `(error, ids) = index.write(docs)
+            pending.signal(list(error, records, ids))
+        }, 0)
+        pending
+    }
+    //
+    // execute
+    //
+    //  loop    operation
+    //    1       scan(a)start
+    //    2       scan(a)end,index(a)start,scan(b)start,index(a)end,update(a)start
+    //    3       scan(b)end,index(b)start,scan(c)start,index(b)end,update(a)end,update(b)start
+    //    4       scan(c)end,index(c)start,scan(d)start,index(c)end,update(b)end,update(c)start
+    //
+    // |-----------|-----------|        |-----------|-----------|
+    // | 1.scan(a) | 3.scan(b) |        | 6.scan(c) | 9.scan(d) |
+    // |-----------|-----------|        |-----------|-----------|
+    //             |------------|------------|      |-------------|
+    //             | 2.index(a) | 5.index(b) |      | 8.index(c)  |
+    //             |------------|------------|      |-------------|
+    //                          |---------------|---------------| |----------------|
+    //                          | 4.update(a,a) | 7.update(b,b) | | 10.update(c,c) |
+    //                          |---------------|---------------| |----------------|
+    //
+    // As of 2023-04-12 with Yandex Cloud and a minimal OpenSearch cluster, The indexing operation is
+    // the long pole in the tent--taking the longest time to process. However the timing has a lot of
+    // variation, e.g. the first DynamoDB update being longer but then very fast after that.
+    //
+    loop {
+        //
+        // finish scanning
+        //
+        if scanPending {
+            `(error, records) = scanPending.wait()
+            scanPending = false
+            if error
+                break
+        }
+        //
+        // start indexing
+        //
+        if records.length > 0
+            indexPending = batchIndex(records)
+        records = false
+        //
+        // start scanning
+        //
+        if !output || output.length <= Number.MAX_SAFE_INTEGER                         // for future limit option
+            scanPending = scanRecords()
+        //
+        // finish indexing
+        //
+        if indexPending {
+            `(error, records, ids) = indexPending.wait()
+            indexPending = false
+            if error
+                break
+            output = output.concat(ids)
+        }
+        //
+        // finish updating
+        //
+        if updatePending {
+            `(data) = updatePending.wait()
+            updatePending = false
+            error = data.find(function(item) { item })
+            if error
+                break
+            options.progress(output.length)                                 // progress with update
+        }
+        //
+        // start updating
+        //
+        if options.writeID {
+            if records && ids
+                updatePending = updateRecordIDs(records, ids, options.writeID)
+        }
+        else
+            options.progress(output.length || 0)                                 // progress without update
+        records = ids = false
+        //
+        // check for termination
+        //
+        if !scanPending && !indexPending && !updatePending
+            break
+    }
+    if error
+       list(error)
+    else
+       list(false, output)
+};
+
+
+/*
+ * osdyInit
+ *
+ *     Initialize the module.
+ *
+ */
+
+function osdyInit(local manifest) {
+    manifest = `(OSDynIndexer, osdyInit)
+
+    Naan.module.build(module.id, "os_dynamo", function(modobj, compobj) {
+        compobj.manifest = manifest
+        require("./openSearch.nlg")
+        modobj.exports.OSDynIndexer = OSDynIndexer
+    })
+} ();