@engine9-io/input-tools 1.8.2 → 1.8.4

package/file/FileUtilities.js

@@ -544,7 +544,7 @@ Worker.prototype.testTransform.metadata = {
 
 /* Get a stream from an actual stream, or an array, or a file */
 Worker.prototype.stream = async function (options) {
-  const { stream: inputStream, packet, type, columns, limit = 100000, filename: filenameOpt } = options;
+  const { stream: inputStream, packet, type, columns, limit, filename: filenameOpt } = options;
   let filename = filenameOpt;
 
   if (inputStream) {
@@ -812,20 +812,30 @@ Worker.prototype.move.metadata = {
 
 Worker.prototype.stat = async function ({ filename }) {
   if (!filename) throw new Error('filename is required');
+  const output = {};
+
+  if (filename.slice(-8) === '.parquet') {
+    const pq = new ParquetWorker(this);
+    output.schema = await pq.schema({ filename });
+    output.records = (await pq.meta({ filename }))?.records;
+  }
+
   if (filename.startsWith('s3://') || filename.startsWith('r2://')) {
     const worker = new (filename.startsWith('r2://') ? R2Worker : S3Worker)(this);
-    return worker.stat({ filename });
+    Object.assign(output, await worker.stat({ filename }));
+  } else {
+    const { ctime, birthtime, size } = await fsp.stat(filename);
+    const modifiedAt = new Date(ctime);
+    let createdAt = birthtime;
+    if (createdAt === 0 || !createdAt) createdAt = ctime;
+    createdAt = new Date(createdAt);
+    Object.assign(output, {
+      createdAt,
+      modifiedAt,
+      size
+    });
   }
-  const { ctime, birthtime, size } = await fsp.stat(filename);
-  const modifiedAt = new Date(ctime);
-  let createdAt = birthtime;
-  if (createdAt === 0 || !createdAt) createdAt = ctime;
-  createdAt = new Date(createdAt);
-  return {
-    createdAt,
-    modifiedAt,
-    size
-  };
+  return output;
 };
 Worker.prototype.stat.metadata = {
   options: {

package/file/Parquet.js

@@ -24,8 +24,11 @@ async function getReader(options) {
 
 Worker.prototype.meta = async function (options) {
   const reader = await getReader(options);
+  const schema = reader.getSchema();
   return {
-    records: String(reader.metadata?.num_rows)
+    //stored as a buffer
+    schema,
+    records: parseInt(reader.metadata?.num_rows?.toString(), 10)
   };
   // getMetadata();
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@engine9-io/input-tools",
-  "version": "1.8.2",
+  "version": "1.8.4",
   "description": "Tools for dealing with Engine9 inputs",
   "main": "index.js",
   "scripts": {

package/timelineTypes.js

@@ -64,6 +64,7 @@ const MESSAGE_DELIVERY_FAILURE_SHOULD_RETRY = 66;
 const MESSAGE_DELIVERY_FAILURE_SHOULD_NOT_RETRY = 66;
 
 const FORM_ADVOCACY = 66;
+const FORM_SURVEY = 67;
 
 const FILE_IMPORT = 70;
 
@@ -128,6 +129,7 @@ const TIMELINE_ENTRY_TYPES = {
   FORM_PETITION,
   FORM_PETITION_CONTACT_TARGET,
   FORM_ADVOCACY,
+  FORM_SURVEY,
 
   MESSAGE_CONVERSION,
   MESSAGE_CONVERSION_ADVOCACY,
@@ -138,10 +140,11 @@ const TIMELINE_ENTRY_TYPES = {
   FILE_IMPORT,
   EXPORT,
   EXPORT_FOR_REMOTE,
-  EXPORT_FOR_MESSAGING,
-
+  EXPORT_FOR_MESSAGING
 };
-Object.entries(TIMELINE_ENTRY_TYPES).forEach(([k, v]) => { TIMELINE_ENTRY_TYPES[v] = k; });
+Object.entries(TIMELINE_ENTRY_TYPES).forEach(([k, v]) => {
+  TIMELINE_ENTRY_TYPES[v] = k;
+});
 module.exports = {
-  TIMELINE_ENTRY_TYPES,
+  TIMELINE_ENTRY_TYPES
 };