@muze-nl/simplystore 0.10.1 → 0.10.2

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@muze-nl/simplystore",
-	"version": "0.10.1",
+	"version": "0.10.2",
 	"main": "src/server.mjs",
 	"type": "module",
 	"scripts": {

package/scripts/convert.mjs

@@ -14,44 +14,73 @@ if (process.argv.length<=3) {
 // parse command line
 let inputFile = process.argv[2]
 let outputFile = process.argv[3]
-let indexFile = process.argv[4] || __dirname+'/../src/index.mjs'
-
-async function main() {
-	// load file
-	let input = fs.readFileSync(inputFile, 'utf-8')
-
-	// parse jsontag
-	let data = JSONTag.parse(input)
-
-	// write resultset to output
-	let strData = stringify(serialize(data))
-	fs.writeFileSync(outputFile, strData)
-	console.log('Converted data written to ',outputFile)
-
-	// now create indexes
-	console.log('Using index library:', indexFile)
-	const index = await import(indexFile).then(mod => {
-	    return mod.default
-	})
-
-	// indexes need the position data which is only available after
-	// parsing the od-jsontag data
-	const parser = new Parser()
-	const odData = parser.parse(strData)
-
-	let meta = {
-		index: {
-			id: new Map()
-		},
-		resultArray: parser.meta.resultArray,
-		data: path.dirname(outputFile)
-	}
-	for (const ob of meta.resultArray) {
-		meta.index.id.set(JSONTag.getAttribute(ob, 'id'), ob)
-	}
+let indexFile = process.argv[4]
+if (indexFile && indexFile[0]!='/') {
+	indexFile = process.cwd()+'/'+indexFile
+} else if (!indexFile) {
+	indexFile = __dirname+'/../src/index.mjs'
+}
+let schemaFile = process.argv[5]
+if (schemaFile && schemaFile[0]!='/') {
+	schemaFile = process.cwd()+'/'+schemaFile
+}
+
+const index = await import(indexFile).then(mod => {
+    return mod.default
+})
+
+// now create indexes
+console.log('Using index library:', indexFile)
+
+
+let schema = {}
+if (schemaFile) {
+	schema = JSONTag.parse(fs.readFileSync(schemaFile, 'utf-8'))
+}
+
+// load file
+let input = fs.readFileSync(inputFile, 'utf-8')
 
-	index.create(odData, meta)
-	console.log('Indexes created')
+// parse jsontag
+let data = JSONTag.parse(input)
+
+console.log('input data parsed')
+// write resultset to output
+let strData = stringify(serialize(data))
+
+console.log('od-jsontag created')
+
+// indexes need the position data which is only available after
+// parsing the od-jsontag data
+const parser = new Parser('https://localhost/',false) // allow mutations
+const odData = parser.parse(strData)
+
+console.log('offsets parsed')
+
+let meta = {
+	index: {
+		id: new Map()
+	},
+	schema,
+	resultArray: parser.meta.resultArray,
+	data: path.dirname(outputFile)
 }
 
-main()
+for (let ob of meta.resultArray) {
+	let id = JSONTag.getAttribute(ob, 'id')
+	if (id) {
+		meta.index.id.set(id, ob)
+	}
+}
+
+console.log('Indexing...')
+//FIXME: offset index can only be calculated by parsing serialized odData
+//but index.create updates that data, so previous information is no longer correct
+//TODO: split index creation in internal (updates odData) and external (creates index files)
+index.create(odData, meta)
+console.log('Indexes created')
+
+strData = stringify(serialize(odData))
+
+fs.writeFileSync(outputFile, strData)
+console.log('Converted data written to ',outputFile)

package/src/server.mjs

@@ -191,8 +191,6 @@ async function main(options) {
         let start = Date.now()
         if (!pool) {
             pool = queryWorkerPool
-        } else {
-            console.log(pool)
         }
         if ( !accept(req,res,
             ['application/jsontag','application/json','text/html','text/javascript','image/*'], 
@@ -224,7 +222,7 @@ async function main(options) {
             request.jsontag = true
         }
         try {
-            let result = await pool.run('query', request)
+            let result = await pool.run('query', request, { timeout })
             sendResponse(result, res)
         } catch(error) {
             sendError(error, res)
@@ -237,10 +235,13 @@ async function main(options) {
         return handleGetQuery(req, res, slowQueryWorkerPool)
     }
 
-    async function handlePostQuery(req,res,pool=null) {
+    async function handlePostQuery(req,res,pool=null, slowTimeout=null) {
         if (!pool) {
             pool = queryWorkerPool
         }
+        if (!slowTimeout) {
+            slowTimeout = timeout
+        }
         let start = Date.now()
         if ( !accept(req,res,
             ['application/jsontag','application/json']) 
@@ -260,7 +261,7 @@ async function main(options) {
             request.jsontag = true
         }
         try {
-            let result = await pool.run('query', request)
+            let result = await pool.run('query', request, {slowTimeout})
             sendResponse(result, res)
         } catch(error) {
             sendError(error, res)
@@ -271,7 +272,7 @@ async function main(options) {
     }
 
     async function handleSlowPostQuery(req, res) {
-        return handlePostQuery(req, res, slowQueryWorkerPool)
+        return handlePostQuery(req, res, slowQueryWorkerPool, slowTimeout)
     }
 
     async function handlePostCommand(req, res) {
@@ -368,13 +369,15 @@ async function main(options) {
                             if (data.data) { // data has changed, commands may do other things instead of changing data
                                 jsontagBuffers.push(data.data) // push changeset to jsontagBuffers so that new query workers get all changes from scratch
                                 Object.assign(meta, data.meta)
-                                queryWorkerPool.update({
+                                const updateTask = {
                                     name: 'update',
                                     req: {
                                         body: jsontagBuffers[jsontagBuffers.length-1], // only add the last change, update tasks for earlier changes have already been sent
                                         meta
                                     }
-                                })
+                                }
+                                queryWorkerPool.update(updateTask)
+                                slowQueryWorkerPool.update(updateTask)
                             }
                             appendFile(commandStatus, JSONTag.stringify(Object.assign({command:command.id}, s)))
                             mainResolve(s)

package/src/workerPool.mjs

@@ -68,9 +68,9 @@ export default class WorkerPool extends EventEmitter {
     worker.postMessage(this.initTask);
   }
 
-  async run(name, req) {
+  async run(name, req, options={}) {
     return new Promise((resolve, reject) => {
-      this.runTask({name,req}, (error, result) => {
+      this.runTask({name,req,...options}, (error, result) => {
         if (error) {
           return reject(error)
         }