@engine9/schemas 1.9.0

package/plugin/schema.js

@@ -0,0 +1,70 @@
+export const tables = [
+  {
+    name: 'plugin',
+    columns: {
+      id: 'id_uuid',
+      path: 'string',
+      name: 'string',
+      nickname: 'string',
+      table_prefix: 'string',
+      remote_plugin_id: 'string',
+      schema: 'json', // Not all plugins need to support file based schemas, like per-account custom fields
+      transforms: 'json', // Not all plugins need to support file based transforms, like per-account custom fields
+      created_at: 'created_at',
+      modified_at: 'modified_at'
+    },
+    indexes: [
+      { columns: 'id', primary: true },
+      { columns: 'remote_plugin_id', unique: true }
+    ]
+  },
+  {
+    name: 'plugin_history',
+    columns: {
+      id: 'id_uuid',
+      plugin_id: 'id_uuid', // can't be null, must have a value
+      path: 'string',
+      created_at: 'created_at',
+      modified_at: 'modified_at'
+    },
+    indexes: [{ columns: 'id', primary: true }]
+  },
+  {
+    name: 'setting',
+    columns: {
+      id: 'id_uuid',
+      plugin_id: 'foreign_uuid',
+      name: 'string',
+      value: 'string',
+      created_at: 'created_at',
+      modified_at: 'modified_at'
+    },
+    indexes: [
+      { columns: 'id', primary: true },
+      { columns: 'plugin_id,name', unique: true }
+    ]
+  },
+  {
+    name: 'input',
+    columns: {
+      id: 'id_uuid',
+      plugin_id: 'id_uuid', // can't be null, must have a value
+      remote_input_id: 'string', // The remote id for this input, unique across the plugin
+      remote_input_name: 'string', // The remote name -- a human name for this input -- e.g. Form Name, message name
+      input_type: 'string', // Type of input, e.g. message, remote_person, petition, signup_form, advocacy_action, etc
+      min_timeline_ts: 'datetime',
+      max_timeline_ts: 'datetime',
+      records: 'bigint', // total timeline records across idv1.parquet files in the input store
+      data_path: 'string', // where the source data lives for this input, a directory or a packet
+      created_at: 'created_at',
+      modified_at: 'modified_at'
+    },
+    indexes: [
+      { columns: 'id', primary: true },
+      { columns: 'plugin_id,remote_input_id', unique: true }
+    ]
+  }
+];
+export default {
+  tables
+};

package/plugin/settings.js

@@ -0,0 +1,19 @@
+export const settings = [
+  {
+    name: 'table_prefix_counter',
+    type: 'int',
+    default: 2729,
+    hidden: true,
+    description: 'Hex table-prefix allocator. Incremented by PluginWorker.getNextTablePrefixCounter.'
+  },
+  {
+    name: 'default_stack',
+    type: 'string',
+    default: '',
+    description:
+      'Stack (or plugin) path for installDefaultPlugins when no path is passed — e.g. @engine9/schemas/stacks/standard or @engine9/schemas/stacks/limited-pii. Empty installs the core person schema plugins only. Ignored when @engine9/schemas/utilities/limited-pii has exclude_pii true (forced limited-pii stack).'
+  }
+];
+export default {
+  settings
+};

package/plugin/ui.console.json5

@@ -0,0 +1,79 @@
+{
+  menu: {
+    settings: {
+      children: {
+        queries: {
+          title: 'Plugins',
+          icon: 'plugins',
+          url: '/plugins'
+        }
+      }
+    }
+  },
+  routes: {
+    plugins: {
+      layout: 'full_width',
+      components: {
+        header: ['Plugins'],
+        main: [
+          {
+            component: 'RecordTable',
+            properties: {
+              table: 'plugin',
+              columns: [
+                {
+                  title: 'Id',
+                  dataIndex: 'id',
+                  sorter: true,
+                  width: '20%'
+                },
+                {
+                  title: 'Name',
+                  dataIndex: 'name',
+                  sorter: true,
+                  template: '{{name}}',
+                  width: '40%'
+                }
+              ],
+              onRecord: {
+                onClick: {
+                  action: 'navigate',
+                  url: '/plugins/{{record.id}}'
+                }
+              }
+            }
+          }
+        ]
+      }
+    },
+    'plugins/:id/*': {
+      layout: 'full_width',
+      components: {
+        main: [
+          {
+            component: 'RecordForm',
+            properties: {
+              table: 'job',
+              title: '{{#if record.id}}Edit Plugin{{else}}Create Job{{/if}}',
+              form: {
+                type: 'object',
+                required: [],
+                properties: {
+                  status: { type: 'string', title: 'Status', default: 'pending' },
+                  options: { type: 'string', title: 'Options', default: '{}' }
+                }
+              },
+              uiSchema: {
+                options: { 'ui:widget': 'textarea' }
+              },
+              onSave: {
+                action: 'navigate',
+                url: '/plugins/{{record.id}}'
+              }
+            }
+          }
+        ]
+      }
+    }
+  }
+}

package/report/index.js

@@ -0,0 +1,7 @@
+const metadata = {
+  name: '@engine9/schemas/report',
+};
+export { metadata };
+export default {
+  metadata
+};

package/report/ui.console.json5

@@ -0,0 +1,21 @@
+{
+  menu: {
+    'data-menu': {
+      children: {
+        report: {
+          title: 'Reports',
+          icon: 'report',
+          url: '/report'
+        }
+      }
+    }
+  },
+  routes: {
+    '/report': {
+      layout: 'full_width',
+      components: {
+        main: ['<h1>Reports</h1>']
+      }
+    }
+  }
+}

package/segment/index.js

@@ -0,0 +1,21 @@
+import schema from './schema.js';
+import search from './search.js';
+import upsert from './transforms/inbound/upsert.js';
+const metadata = {
+  name: '@engine9/schemas/segment',
+  dependencies: {
+    '@engine9/schemas/person': '>=1.9.0'
+  }
+};
+export const transforms = {
+  upsert
+};
+export { metadata };
+export { schema };
+export { search };
+export default {
+  metadata,
+  schema,
+  search,
+  transforms
+};

package/segment/schema.js

@@ -0,0 +1,81 @@
+export const tables = [
+  {
+    name: 'segment_folder',
+    columns: {
+      id: 'id_uuid',
+      name: 'string',
+      description: 'text',
+      created_at: 'created_at',
+      modified_at: 'modified_at'
+    },
+    indexes: [{ columns: 'id', primary: true }]
+  },
+  {
+    name: 'segment',
+    columns: {
+      id: 'id_uuid',
+      plugin_id: 'id_uuid', // can't be null, must have a value
+      submodule: 'string',
+      segment_folder_id: 'foreign_uuid',
+      remote_segment_id: 'string',
+      legacy_id: 'int',
+      name: 'string',
+      category: 'string', // A arbitrary type, user facing, not to be used for logic
+      definition_path: 'string', // a definition file path, will be used for configuring
+      stars: 'int', //number of stars, good for UI organizing
+      search: 'json',
+      build_type: {
+        type: 'enum',
+        nullable: false,
+        default_value: 'list',
+        values: [
+          'list', //static list of people
+          'search', // built by the defined search
+          'scheduled_search', // built on a schedule by the search
+          'remote_count', // no person records, just counts, pulled from remote site
+          'remote', // pulled from remote plugin site
+          'manual' // created outside normal processing
+        ]
+      },
+      build_schedule: 'string',
+      build_status: 'string',
+      build_status_modified_at: 'modified_at',
+      build_table: 'string',
+      last_built: 'datetime',
+      // Membership policy. null = only a request with `admin` scope.
+      // join_min_level:  lowest Identity Level at which a person may add
+      //                  themselves (0 = a public form, no login)
+      // leave_min_level: lowest Identity Level at which a person may remove
+      //                  themselves (never 0)
+      // manager_role_id: role segment whose members may add and remove
+      //                  other people in this segment
+      // Enforced by @engine9/core (docs/segments.md), not by the database.
+      join_min_level: { type: 'int', nullable: true },
+      leave_min_level: { type: 'int', nullable: true },
+      manager_role_id: 'foreign_uuid',
+      people: 'int', // engine9 count
+      reported_people: 'int', // reported by outside parties
+      created_at: 'created_at',
+      modified_at: 'modified_at'
+    },
+    indexes: [
+      { columns: 'id', primary: true },
+      // if there is a remote_segment_id, it should be unique for the plugin
+      { columns: ['plugin_id', 'remote_segment_id'], unique: true }
+    ]
+  },
+  // Used for some smaller segments, and a cache of segments
+  {
+    name: 'person_segment',
+    columns: {
+      id: 'id',
+      person_id: 'person_id',
+      segment_id: 'foreign_uuid',
+      modified_at: 'modified_at'
+    },
+    indexes: [{ columns: 'segment_id,person_id', unique: true }, { columns: 'person_id' }]
+  }
+];
+export default {
+  tables
+};

package/segment/search.js

@@ -0,0 +1,64 @@
+export const segment = {
+  title: 'Segment',
+  form: {
+    title: 'Segment',
+    type: 'object',
+    properties: {
+      segmentId: {
+        title: 'Segment',
+        description: '',
+        type: 'string'
+      }
+    },
+    required: []
+  },
+  optionsToEQLContext: function ({ segmentId }) {
+    return {
+      segments: {
+        table: 'segment',
+        columns: ['id', 'name'],
+        conditions: [
+          {
+            type: 'EQUALS',
+            values: [
+              {
+                ref: { column: 'id' }
+              },
+              {
+                value: { value: segmentId }
+              }
+            ]
+          }
+        ]
+      }
+    };
+  },
+  /* map from provided user data into an EQL structure */
+  optionsToEQL(options, { segments } = {}) {
+    const { segmentId } = options;
+    const segment = segments.find(({ id }) => id == segmentId);
+    if (!segment) throw new Error('Could not find segment ' + segmentId + ' in ' + segments.length + ' segments');
+    let text = 'In ' + segment.name || segmentId;
+    const conditions = [
+      {
+        type: 'EQUALS',
+        values: [
+          {
+            ref: { column: 'segment_id' }
+          },
+          {
+            value: { value: segmentId }
+          }
+        ]
+      }
+    ];
+    const eql = {
+      table: 'person_segment',
+      conditions
+    };
+    return { text, eql };
+  }
+};
+export default {
+  segment
+};

package/segment/transforms/inbound/upsert.js

@@ -0,0 +1,44 @@
+import { mergeIntoQueue } from '@engine9/input-tools';
+
+export const bindings = {
+  tablesToUpsert: { path: 'sql.tables.upsert' },
+  uuidIsValid: { path: '@engine9/input-tools:uuidIsValid' }
+};
+export async function transform({ batch, tablesToUpsert, uuidIsValid, options }) {
+  const { segmentIds } = options;
+  const globalSegments = String(segmentIds || '')
+    .split(',')
+    .map((d) => d.trim())
+    .filter(Boolean);
+  batch.forEach((o) => {
+    const localSegmentIds = String(o.segment_ids || '')
+      .split(',')
+      .map((d) => d.trim())
+      .filter(Boolean);
+    const allSegmentIds = [].concat(globalSegments).concat(localSegmentIds);
+    if (allSegmentIds.length === 0) return;
+    const invalid = allSegmentIds.filter((uuid) => !uuidIsValid(uuid));
+    if (invalid.length > 0) throw new Error(`There are some invalid segment_ids:${invalid.join(',')}`);
+    tablesToUpsert.person_segment = tablesToUpsert.person_segment || [];
+    allSegmentIds.forEach((sid) =>
+      mergeIntoQueue(
+        tablesToUpsert.person_segment,
+        {
+          id: null,
+          person_id: o.person_id,
+          segment_id: sid
+        },
+        {
+          keyFields: ['segment_id', 'person_id'],
+          merge: (existing) => existing,
+          label: 'person_segment'
+        }
+      )
+    );
+  });
+  return batch;
+}
+export default {
+  bindings,
+  transform
+};

package/segment/ui.console.json5

@@ -0,0 +1,142 @@
+{
+  menu: {
+    people: {
+      children: {
+        segments: {
+          title: 'Segments',
+          icon: 'people',
+          url: '/segments'
+        }
+      }
+    }
+  },
+  routes: {
+    '/segments': {
+      layout: 'full_width',
+      components: {
+        header: [
+          'Segments',
+          {
+            component: 'Button',
+            properties: {
+              icon: 'plus',
+              content: 'Add Segment',
+              onClick: {
+                action: 'navigate',
+                url: '/segments/create'
+              }
+            }
+          }
+        ],
+        main: [
+          {
+            component: 'RecordTable',
+            properties: {
+              table: 'segment',
+              columns: [
+                {
+                  title: 'Id',
+                  dataIndex: 'id',
+                  sorter: true,
+                  width: '20%'
+                },
+                {
+                  title: 'Type',
+                  dataIndex: 'type',
+                  sorter: true
+                },
+                {
+                  title: 'Name',
+                  dataIndex: 'label',
+                  sorter: true,
+                  template: '{{label}}',
+                  width: '40%'
+                }
+              ],
+              onRecord: {
+                onClick: {
+                  action: 'navigate',
+                  url: '/segments/{{record.id}}'
+                }
+              }
+            }
+          }
+        ]
+      }
+    },
+    '/segments/:id/*': {
+      layout: 'full_width',
+      components: {
+        main: [
+          {
+            component: 'RecordForm',
+            properties: {
+              table: 'job',
+              title: '{{#if record.id}}Edit Segment{{else}}Create Segment{{/if}}',
+              form: {
+                type: 'object',
+                required: [],
+                properties: {
+                  type: { type: 'string', title: 'Type' },
+                  name: { type: 'string', title: 'Name' },
+                  plugin_id: { type: 'string', title: 'Source Plugin ID' },
+                  remote_segment_id: { type: 'string', title: 'Remote Segment ID' },
+                  build_mechanism: { type: 'string', title: 'Mechanism', default: 'static' },
+                  build_schedule: { type: 'string', title: 'Build Schedule' },
+                  build_status: { type: 'string', title: 'Build Status' },
+                  build_status_modified_at: { type: 'modified_at', title: 'Build Status Modified At' }
+                }
+              },
+              uiSchema: {},
+              onSave: {
+                action: 'navigate',
+                url: '/segments/{{record.id}}'
+              }
+            }
+          },
+          {
+            component: 'RecordTable',
+            properties: {
+              table: 'person_segment',
+              includes: [{ alias: 'people', table: 'person' }],
+              conditions: [{ eql: 'segment_id={{parameters.segment_id}}' }],
+              columns: [
+                {
+                  template: '{{person_id}}'
+                }
+              ]
+            }
+          }
+        ]
+      }
+    },
+    '/person/:id/*': {
+      components: {
+        sidebar: {
+          items2: [
+            {
+              component: 'Link',
+              properties: {
+                label: 'Segments',
+                to: 'segments'
+              }
+            }
+          ]
+        },
+        main: [
+          {
+            path: 'segments',
+            label: 'Segments',
+            component: 'RecordTable',
+            properties: {
+              table: 'person_segment',
+              includes: [{ alias: 'segments', table: 'segment' }],
+              conditions: [{ eql: 'person_id={{parameters.person_id}}' }],
+              rows: ['{{segment}}']
+            }
+          }
+        ]
+      }
+    }
+  }
+}

package/segment_stats/index.js

@@ -0,0 +1,16 @@
+import schema from './schema.js';
+
+const metadata = {
+  name: '@engine9/schemas/segment_stats',
+  schemas: ['schema.js'],
+  dependencies: {
+    '@engine9/schemas/segment': '>=1.9.0'
+  }
+};
+
+export { metadata };
+export { schema };
+export default {
+  metadata,
+  schema
+};

package/segment_stats/schema.js

@@ -0,0 +1,30 @@
+export const tables = [
+  {
+    name: 'segment_stats',
+    columns: {
+      id: 'id',
+      segment_id: 'foreign_id',
+      transactions: 'int',
+      revenue: 'currency',
+      created_at: 'created_at',
+      modified_at: 'modified_at'
+    },
+    indexes: [{ columns: ['segment_id'] }]
+  },
+  {
+    name: 'segment_stats_by_date',
+    columns: {
+      id: 'id',
+      segment_id: 'foreign_id',
+      date: 'date',
+      transactions: 'int',
+      revenue: 'currency',
+      created_at: 'created_at',
+      modified_at: 'modified_at'
+    },
+    indexes: [{ columns: ['segment_id', 'date'], unique: true }]
+  }
+];
+export default {
+  tables
+};

package/source_code/index.js

@@ -0,0 +1,16 @@
+import schema from './schema.js';
+import metrics from './metrics.js';
+const metadata = {
+  name: '@engine9/schemas/source_code',
+  dependencies: {
+    '@engine9/schemas/person': '>=1.9.0'
+  }
+};
+export { metadata };
+export { schema };
+export { metrics };
+export default {
+  metadata,
+  schema,
+  metrics
+};

package/source_code/metrics.js

@@ -0,0 +1,28 @@
+export const parsing = {
+  label: 'Overall Parsing',
+  description: '',
+  eql: {
+    table: 'source_code_summary',
+    columns: [
+      { eql: 'count(*)', name: 'records' },
+      { eql: 'sum(case when length(format)>0 then 1 else 0 end)', name: 'parsed' }
+    ],
+    conditions: []
+  }
+};
+export const recent_parsing = {
+  label: 'Parsing last 30 days',
+  description: '',
+  eql: {
+    table: 'source_code_summary',
+    columns: [
+      { eql: 'count(*)', name: 'records' },
+      { eql: 'sum(case when length(format)>0 then 1 else 0 end)', name: 'parsed' }
+    ],
+    conditions: [{ eql: 'date_sub(now(), interval 30 day)' }]
+  }
+};
+export default {
+  parsing,
+  recent_parsing
+};

package/source_code/schema.js

@@ -0,0 +1,29 @@
+export const tables = [
+  {
+    name: 'source_code_dictionary',
+    columns: {
+      source_code_id: 'id',
+      source_code: {
+        type: 'string',
+        length: 180,
+        nullable: false,
+        default_value: ''
+      },
+      format: 'string',
+      format_regex: 'string',
+      source_code_channel: 'string',
+      source_code_last_used: 'created_at', //by default, when this record is created it needs a last used.
+      parsing: 'json',
+      created_at: 'created_at',
+      modified_at: 'modified_at'
+    },
+    indexes: [
+      { columns: 'source_code_id', primary: true },
+      { columns: 'source_code', unique: true },
+      { columns: 'source_code_last_used' }
+    ]
+  }
+];
+export default {
+  tables
+};