@engine9/schemas 1.9.1 → 1.10.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +2 -2
- package/person/index.js +6 -20
- package/person/schema.js +0 -2
- package/person/ui.console.json5 +14 -71
- package/person_hash/README.md +3 -3
- package/person_name/README.md +69 -0
- package/person_name/index.js +28 -0
- package/person_name/schema.js +17 -0
- package/person_name/search.js +47 -0
- package/person_name/transforms/inbound/upsert_tables.js +63 -0
- package/{person → person_name}/transforms/outbound/appendName.js +5 -9
- package/person_name/upsert.test.js +91 -0
- package/stacks/limited-pii/stack.json +5 -4
- package/stacks/stacks.test.js +6 -0
- package/stacks/standard/stack.json +3 -2
- package/transaction/contact_details/schema.js +3 -1
- package/transaction/core/README.md +2 -1
- package/transaction/core/schema.js +0 -3
- package/utilities/limited-pii/settings.js +1 -1
package/package.json
CHANGED
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@engine9/schemas",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.10.1",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"description": "Engine9 standard schemas: shared tables, transforms, searches, and segments",
|
|
6
6
|
"main": "index.js",
|
|
7
7
|
"scripts": {
|
|
8
|
-
"test": "node --test person/upsert.test.js person_address/normalize.test.js person_email/upsert.test.js person_hash/hash.test.js person_phone/upsert.test.js stacks/stacks.test.js transaction/core/attribute.test.js"
|
|
8
|
+
"test": "node --test person/upsert.test.js person_address/normalize.test.js person_email/upsert.test.js person_hash/hash.test.js person_name/upsert.test.js person_phone/upsert.test.js stacks/stacks.test.js transaction/core/attribute.test.js"
|
|
9
9
|
},
|
|
10
10
|
"engines": {
|
|
11
11
|
"node": ">=18"
|
package/person/index.js
CHANGED
|
@@ -2,21 +2,23 @@ import schema from './schema.js';
|
|
|
2
2
|
import upsertPerson from './transforms/inbound/upsert_tables.js';
|
|
3
3
|
import normalizeFieldNames from './transforms/inbound/normalize_field_names.js';
|
|
4
4
|
import simpleMap from './transforms/simpleMap.js';
|
|
5
|
-
import appendName from '
|
|
5
|
+
import appendName from '../person_name/transforms/outbound/appendName.js';
|
|
6
6
|
import metrics from './metrics.js';
|
|
7
7
|
const metadata = {
|
|
8
8
|
name: '@engine9/schemas/person',
|
|
9
9
|
schemas: ['schema.js'],
|
|
10
10
|
// Inbound people pipeline slots -> transform export keys (woven by core when installed)
|
|
11
11
|
inbound: {
|
|
12
|
-
normalize: ['normalizeFieldNames']
|
|
13
|
-
upsert: ['upsertPerson']
|
|
12
|
+
normalize: ['normalizeFieldNames']
|
|
14
13
|
}
|
|
15
14
|
};
|
|
16
15
|
const transforms = {
|
|
17
16
|
normalizeFieldNames,
|
|
17
|
+
// Not woven. Plugin rows installed before person_name still name it in their
|
|
18
|
+
// inbound snapshot until the person_name upgrade refreshes that snapshot.
|
|
18
19
|
upsertPerson,
|
|
19
20
|
simpleMap,
|
|
21
|
+
// Saved export definitions still call @engine9/schemas/person:transforms:appendName.
|
|
20
22
|
appendName
|
|
21
23
|
};
|
|
22
24
|
export const search = {
|
|
@@ -26,14 +28,6 @@ export const search = {
|
|
|
26
28
|
title: 'Primary Info',
|
|
27
29
|
type: 'object',
|
|
28
30
|
properties: {
|
|
29
|
-
givenName: {
|
|
30
|
-
title: 'First Name',
|
|
31
|
-
type: 'string'
|
|
32
|
-
},
|
|
33
|
-
familyName: {
|
|
34
|
-
title: 'Last Name',
|
|
35
|
-
type: 'string'
|
|
36
|
-
},
|
|
37
31
|
ids: {
|
|
38
32
|
title: 'Person Ids (comma delimited)',
|
|
39
33
|
type: 'string'
|
|
@@ -51,7 +45,7 @@ export const search = {
|
|
|
51
45
|
},
|
|
52
46
|
optionsToEQL: (options) => {
|
|
53
47
|
const text = [];
|
|
54
|
-
const { ids, idGreaterThanEqual, idLessThan
|
|
48
|
+
const { ids, idGreaterThanEqual, idLessThan } = options;
|
|
55
49
|
const conditions = [];
|
|
56
50
|
if (ids) {
|
|
57
51
|
let arr = ids || '';
|
|
@@ -68,14 +62,6 @@ export const search = {
|
|
|
68
62
|
conditions.push({ eql: `id<${idLessThan}` });
|
|
69
63
|
text.push(`Person ID less than ${idLessThan}`);
|
|
70
64
|
}
|
|
71
|
-
if (givenName.length > 0) {
|
|
72
|
-
text.push(`First name like '${givenName}'`);
|
|
73
|
-
conditions.push({ eql: `given_name like '${givenName}'` });
|
|
74
|
-
}
|
|
75
|
-
if (familyName.length > 0) {
|
|
76
|
-
text.push(`Last name like '${givenName}'`);
|
|
77
|
-
conditions.push({ eql: `family_name like '${familyName}'` });
|
|
78
|
-
}
|
|
79
65
|
return {
|
|
80
66
|
text,
|
|
81
67
|
eql: {
|
package/person/schema.js
CHANGED
package/person/ui.console.json5
CHANGED
|
@@ -17,26 +17,16 @@
|
|
|
17
17
|
person: {
|
|
18
18
|
layout: 'full_width',
|
|
19
19
|
components: {
|
|
20
|
-
header: [
|
|
21
|
-
'Everybody',
|
|
22
|
-
{
|
|
23
|
-
component: 'Button',
|
|
24
|
-
properties: {
|
|
25
|
-
icon: 'plus',
|
|
26
|
-
content: 'Add Person',
|
|
27
|
-
onClick: {
|
|
28
|
-
action: 'navigate',
|
|
29
|
-
url: '/person/create'
|
|
30
|
-
}
|
|
31
|
-
}
|
|
32
|
-
}
|
|
33
|
-
],
|
|
20
|
+
header: ['Everybody'],
|
|
34
21
|
main: [
|
|
35
22
|
{
|
|
36
23
|
component: 'RecordList',
|
|
37
24
|
properties: {
|
|
38
25
|
table: 'person',
|
|
39
26
|
includes: {
|
|
27
|
+
names: {
|
|
28
|
+
table: 'person_name'
|
|
29
|
+
},
|
|
40
30
|
emails: {
|
|
41
31
|
table: 'person_email'
|
|
42
32
|
}
|
|
@@ -51,8 +41,7 @@
|
|
|
51
41
|
{
|
|
52
42
|
title: 'Name',
|
|
53
43
|
dataIndex: 'name',
|
|
54
|
-
|
|
55
|
-
template: '{{given_name}} {{family_name}} {{emails.0.email}}',
|
|
44
|
+
template: '{{names.0.given_name}} {{names.0.family_name}} {{emails.0.email}}',
|
|
56
45
|
width: '40%'
|
|
57
46
|
},
|
|
58
47
|
{
|
|
@@ -72,36 +61,6 @@
|
|
|
72
61
|
]
|
|
73
62
|
}
|
|
74
63
|
},
|
|
75
|
-
'person/create': {
|
|
76
|
-
layout: 'full_width',
|
|
77
|
-
components: {
|
|
78
|
-
main: [
|
|
79
|
-
{
|
|
80
|
-
path: 'edit',
|
|
81
|
-
component: 'RecordForm',
|
|
82
|
-
properties: {
|
|
83
|
-
table: 'person',
|
|
84
|
-
form: {
|
|
85
|
-
title: 'Todo',
|
|
86
|
-
type: 'object',
|
|
87
|
-
required: [],
|
|
88
|
-
properties: {
|
|
89
|
-
given_name: { type: 'string', title: 'First Name', default: '' },
|
|
90
|
-
family_name: { type: 'string', title: 'Last Name', default: '' }
|
|
91
|
-
}
|
|
92
|
-
},
|
|
93
|
-
uiSchema: {
|
|
94
|
-
family_name: { 'ui:widget': 'textarea' }
|
|
95
|
-
},
|
|
96
|
-
onSave: {
|
|
97
|
-
action: 'navigate',
|
|
98
|
-
url: '/person/{{record.id}}'
|
|
99
|
-
}
|
|
100
|
-
}
|
|
101
|
-
}
|
|
102
|
-
]
|
|
103
|
-
}
|
|
104
|
-
},
|
|
105
64
|
'person/:id/*': {
|
|
106
65
|
layout: 'sidebar',
|
|
107
66
|
components: {
|
|
@@ -114,19 +73,15 @@
|
|
|
114
73
|
table: 'person',
|
|
115
74
|
id: '{{parameters.id}}',
|
|
116
75
|
includes: {
|
|
76
|
+
names: {
|
|
77
|
+
table: 'person_name'
|
|
78
|
+
},
|
|
117
79
|
emails: {
|
|
118
80
|
table: 'person_email',
|
|
119
81
|
orderBy: 'preference_order'
|
|
120
82
|
}
|
|
121
83
|
},
|
|
122
|
-
template: '{{given_name}} {{family_name}} {{#if emails.0.email}}<{{emails.0.email}}>{{/if}}'
|
|
123
|
-
}
|
|
124
|
-
},
|
|
125
|
-
{
|
|
126
|
-
component: 'Link',
|
|
127
|
-
properties: {
|
|
128
|
-
label: 'Edit',
|
|
129
|
-
to: 'edit'
|
|
84
|
+
template: '{{names.0.given_name}} {{names.0.family_name}} {{#if emails.0.email}}<{{emails.0.email}}>{{/if}}'
|
|
130
85
|
}
|
|
131
86
|
}
|
|
132
87
|
]
|
|
@@ -138,24 +93,12 @@
|
|
|
138
93
|
properties: {
|
|
139
94
|
table: 'person',
|
|
140
95
|
id: '{{parameters.id}}',
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
{
|
|
145
|
-
path: 'edit',
|
|
146
|
-
component: 'RecordForm',
|
|
147
|
-
properties: {
|
|
148
|
-
table: 'person',
|
|
149
|
-
id: '{{parameters.id}}',
|
|
150
|
-
form: {
|
|
151
|
-
title: 'Todo',
|
|
152
|
-
type: 'object',
|
|
153
|
-
required: [],
|
|
154
|
-
properties: {
|
|
155
|
-
given_name: { type: 'string', title: 'First Name', default: '' },
|
|
156
|
-
family_name: { type: 'string', title: 'Last Name', default: '' }
|
|
96
|
+
includes: {
|
|
97
|
+
names: {
|
|
98
|
+
table: 'person_name'
|
|
157
99
|
}
|
|
158
|
-
}
|
|
100
|
+
},
|
|
101
|
+
template: '{{names.0.given_name}} {{names.0.family_name}}'
|
|
159
102
|
}
|
|
160
103
|
}
|
|
161
104
|
]
|
package/person_hash/README.md
CHANGED
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
|
|
5
5
|
It is **not** part of the standard stack. It ships in `@engine9/schemas/stacks/limited-pii`, or install it explicitly (once per account — `unique: true`). Once installed, core weaves its transforms into the inbound people pipeline automatically.
|
|
6
6
|
|
|
7
|
-
Warehouse plugin setting `exclude_pii` on `@engine9/schemas/utilities/limited-pii` makes `installDefaultPlugins()` prefer the limited-pii stack over `default_stack` and refuses `@engine9/schemas/stacks/standard`, `person_email`, `person_phone`, and `
|
|
7
|
+
Warehouse plugin setting `exclude_pii` on `@engine9/schemas/utilities/limited-pii` makes `installDefaultPlugins()` prefer the limited-pii stack over `default_stack` and refuses `@engine9/schemas/stacks/standard`, `person_name`, `person_email`, `person_phone`, `person_address`, and `transaction/contact_details` even if those plugins are already installed. That does **not** uninstall leftover plaintext tables. Install the utility plugin on demand; it is not part of the limited-pii stack.
|
|
8
8
|
|
|
9
9
|
It does **not** depend on `person_email` or `person_phone`. Those plugins may be installed alongside it; this schema plugin never writes to their tables.
|
|
10
10
|
|
|
@@ -31,7 +31,7 @@ inbound: {
|
|
|
31
31
|
|
|
32
32
|
How slots and the weaver work: `@engine9/core/lib/peoplePipeline/README.md`.
|
|
33
33
|
|
|
34
|
-
To see the woven chain for an account: `personWorker.getInboundTransforms({ pluginId, describe: true })` (CLI: `e9 personworker getInboundTransforms --plugin_id=<plugin_id> --describe=true`). On a limited-pii account the listing has `person_hash` on `id` and `upsert` and no `person_email` / `person_phone` / `person_address` lines. Drop a step for one job with `omit_transforms`.
|
|
34
|
+
To see the woven chain for an account: `personWorker.getInboundTransforms({ pluginId, describe: true })` (CLI: `e9 personworker getInboundTransforms --plugin_id=<plugin_id> --describe=true`). On a limited-pii account the listing has `person_hash` on `id` and `upsert` and no `person_name` / `person_email` / `person_phone` / `person_address` lines. Drop a step for one job with `omit_transforms`.
|
|
35
35
|
|
|
36
36
|
## Data Model
|
|
37
37
|
|
|
@@ -67,4 +67,4 @@ The `upsertPersonHash` transform writes only hash columns to `person_hash_email`
|
|
|
67
67
|
|
|
68
68
|
The plugin also exports `search.emailHashes` / `search.phoneHashes` for the plugin search tree (`@engine9/schemas/person_hash:search:emailHashes`).
|
|
69
69
|
|
|
70
|
-
Default warehouse export includes the hash tables (export skips them when they do not exist). When utilities/limited-pii `exclude_pii` is set and `tables` is not explicit, `person_email` / `person_phone` / `person_address` are omitted from that default list.
|
|
70
|
+
Default warehouse export includes the hash tables (export skips them when they do not exist). When utilities/limited-pii `exclude_pii` is set and `tables` is not explicit, `person_name` / `person_email` / `person_phone` / `person_address` are omitted from that default list.
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
# Person Name Schema Plugin
|
|
2
|
+
|
|
3
|
+
`@engine9/schemas/person_name` stores a person's plaintext name: `given_name` and `family_name`, one row per person.
|
|
4
|
+
|
|
5
|
+
Names are personal data, so they are not on `person`. `person` holds only the id and timestamps, and every account installs it. An account that must not store names leaves this plugin out, the same way it leaves out `person_email`, `person_phone`, and `person_address`.
|
|
6
|
+
|
|
7
|
+
- The standard stack (`@engine9/schemas/stacks/standard`) and the no-argument `installDefaultPlugins()` install it.
|
|
8
|
+
- The limited-pii stack (`@engine9/schemas/stacks/limited-pii`) excludes it. With `exclude_pii` set on `@engine9/schemas/utilities/limited-pii`, core refuses to install it.
|
|
9
|
+
|
|
10
|
+
## Install
|
|
11
|
+
|
|
12
|
+
```javascript
|
|
13
|
+
await pluginWorker.install({ path: '@engine9/schemas/person_name' });
|
|
14
|
+
```
|
|
15
|
+
|
|
16
|
+
A second install of the same path reuses the existing plugin row.
|
|
17
|
+
|
|
18
|
+
## Inbound pipeline slots
|
|
19
|
+
|
|
20
|
+
```javascript
|
|
21
|
+
inbound: {
|
|
22
|
+
upsert: ['upsertPersonName']
|
|
23
|
+
}
|
|
24
|
+
```
|
|
25
|
+
|
|
26
|
+
When the plugin is installed, core weaves `upsertPersonName` into the `upsert` slot of the inbound people pipeline. When it is not installed, the step is not in the chain. Incoming `given_name` / `family_name` fields are then ignored and nothing is stored. How slots and the weaver work: `@engine9/core/lib/peoplePipeline/README.md`.
|
|
27
|
+
|
|
28
|
+
## Data Model
|
|
29
|
+
|
|
30
|
+
### `person_name`
|
|
31
|
+
|
|
32
|
+
Primary key: `person_id`.
|
|
33
|
+
|
|
34
|
+
| Field | Type | Description |
|
|
35
|
+
| --- | --- | --- |
|
|
36
|
+
| `person_id` | `person_id` | Person the name belongs to. |
|
|
37
|
+
| `given_name` | `string` | Given (first) name. |
|
|
38
|
+
| `family_name` | `string` | Family (last) name. |
|
|
39
|
+
| `source_input_id` | `foreign_uuid` | Input that first wrote the row. Existing rows keep their original source. Rows copied by the core `person_name` upgrade have none. |
|
|
40
|
+
| `created_at` / `modified_at` | timestamps | |
|
|
41
|
+
|
|
42
|
+
## Inbound Behavior
|
|
43
|
+
|
|
44
|
+
`upsertPersonName` runs for each row that has a `given_name` or `family_name` key:
|
|
45
|
+
|
|
46
|
+
- A key that is present overwrites the stored value. An explicit `null` clears it.
|
|
47
|
+
- A key that is absent keeps the stored value.
|
|
48
|
+
- Rows for the same person in one batch merge; later keys win.
|
|
49
|
+
- A row with a name and no `person_id` fails the batch.
|
|
50
|
+
|
|
51
|
+
## Outbound Behavior
|
|
52
|
+
|
|
53
|
+
`appendName` adds `given_name` and `family_name` to each row by `person_id`. People with no row get `null` for both.
|
|
54
|
+
|
|
55
|
+
`@engine9/schemas/person:transforms:appendName` is the same transform, kept so saved export definitions keep working.
|
|
56
|
+
|
|
57
|
+
## Search
|
|
58
|
+
|
|
59
|
+
`names` matches people by `given_name` and/or `family_name` with `LIKE`. Searching with neither finds everyone who has a name row.
|
|
60
|
+
|
|
61
|
+
## Upgrading accounts created before this plugin
|
|
62
|
+
|
|
63
|
+
Before this plugin, the names were `person.given_name` and `person.family_name`. Run the core upgrade once per account after installing this version of `@engine9/schemas`:
|
|
64
|
+
|
|
65
|
+
- `PluginWorker.upgrade()` (server: `e9 personworker upgrade -a <account_id>`)
|
|
66
|
+
- `npx e9core upgrade --db <connection>` for a SQLite or MySQL database
|
|
67
|
+
- `npx e9core upgrade --url https://<site>` for a Cloudflare site (it posts to `/api/admin/upgrade` with `E9_ADMIN_API_KEY`)
|
|
68
|
+
|
|
69
|
+
The upgrade installs this plugin and copies every non-blank name into `person_name`. It then refreshes the `person` plugin's inbound snapshot and drops both columns from `person`. On a PII-free account it does not install this plugin. It drops the columns only when they are empty, or when you pass `discard_names`.
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import schema from './schema.js';
|
|
2
|
+
import upsertPersonName from './transforms/inbound/upsert_tables.js';
|
|
3
|
+
import appendName from './transforms/outbound/appendName.js';
|
|
4
|
+
import search from './search.js';
|
|
5
|
+
const metadata = {
|
|
6
|
+
name: '@engine9/schemas/person_name',
|
|
7
|
+
unique: true,
|
|
8
|
+
dependencies: {
|
|
9
|
+
'@engine9/schemas/person': '>=1.9.0'
|
|
10
|
+
},
|
|
11
|
+
// Inbound people pipeline slots -> transform export keys (woven by core when installed)
|
|
12
|
+
inbound: {
|
|
13
|
+
upsert: ['upsertPersonName']
|
|
14
|
+
}
|
|
15
|
+
};
|
|
16
|
+
export const transforms = {
|
|
17
|
+
upsertPersonName,
|
|
18
|
+
appendName
|
|
19
|
+
};
|
|
20
|
+
export { metadata };
|
|
21
|
+
export { schema };
|
|
22
|
+
export { search };
|
|
23
|
+
export default {
|
|
24
|
+
metadata,
|
|
25
|
+
schema,
|
|
26
|
+
search,
|
|
27
|
+
transforms
|
|
28
|
+
};
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
export const tables = [
|
|
2
|
+
{
|
|
3
|
+
name: 'person_name',
|
|
4
|
+
columns: {
|
|
5
|
+
person_id: 'person_id',
|
|
6
|
+
given_name: 'string',
|
|
7
|
+
family_name: 'string',
|
|
8
|
+
source_input_id: 'foreign_uuid',
|
|
9
|
+
created_at: 'created_at',
|
|
10
|
+
modified_at: 'modified_at'
|
|
11
|
+
},
|
|
12
|
+
indexes: [{ columns: ['person_id'], primary: true }]
|
|
13
|
+
}
|
|
14
|
+
];
|
|
15
|
+
export default {
|
|
16
|
+
tables
|
|
17
|
+
};
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
export const names = {
|
|
2
|
+
title: 'Names',
|
|
3
|
+
form: {
|
|
4
|
+
title: 'Names',
|
|
5
|
+
type: 'object',
|
|
6
|
+
properties: {
|
|
7
|
+
givenName: {
|
|
8
|
+
title: 'First Name',
|
|
9
|
+
type: 'string'
|
|
10
|
+
},
|
|
11
|
+
familyName: {
|
|
12
|
+
title: 'Last Name',
|
|
13
|
+
type: 'string'
|
|
14
|
+
}
|
|
15
|
+
},
|
|
16
|
+
required: []
|
|
17
|
+
},
|
|
18
|
+
optionsToEQL(options) {
|
|
19
|
+
const { givenName = '', familyName = '' } = options;
|
|
20
|
+
const text = [];
|
|
21
|
+
const conditions = [];
|
|
22
|
+
if (givenName.length > 0) {
|
|
23
|
+
text.push(`First name like '${givenName}'`);
|
|
24
|
+
conditions.push({
|
|
25
|
+
type: 'LIKE',
|
|
26
|
+
values: [{ ref: { column: 'given_name' } }, { value: { value: givenName } }]
|
|
27
|
+
});
|
|
28
|
+
}
|
|
29
|
+
if (familyName.length > 0) {
|
|
30
|
+
text.push(`Last name like '${familyName}'`);
|
|
31
|
+
conditions.push({
|
|
32
|
+
type: 'LIKE',
|
|
33
|
+
values: [{ ref: { column: 'family_name' } }, { value: { value: familyName } }]
|
|
34
|
+
});
|
|
35
|
+
}
|
|
36
|
+
return {
|
|
37
|
+
text: text.length ? text.join(' and ') : 'Has a name',
|
|
38
|
+
eql: {
|
|
39
|
+
table: 'person_name',
|
|
40
|
+
conditions
|
|
41
|
+
}
|
|
42
|
+
};
|
|
43
|
+
}
|
|
44
|
+
};
|
|
45
|
+
export default {
|
|
46
|
+
names
|
|
47
|
+
};
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
import { mergeIntoQueue } from '@engine9/input-tools';
|
|
2
|
+
|
|
3
|
+
export const type = 'upsert';
|
|
4
|
+
export const description =
|
|
5
|
+
'Upsert given_name / family_name into person_name, one row per person. A name key that is absent keeps the stored value; an explicit null clears it.';
|
|
6
|
+
export const bindings = {
|
|
7
|
+
databaseNames: { path: 'sql.query', options: { table: 'person_name', lookup: ['person_id'] } },
|
|
8
|
+
tablesToUpsert: { path: 'sql.tables.upsert' }
|
|
9
|
+
};
|
|
10
|
+
|
|
11
|
+
const NAME_COLUMNS = ['given_name', 'family_name'];
|
|
12
|
+
|
|
13
|
+
// person_id may arrive as number or string from different pipeline stages
|
|
14
|
+
const personKey = (field, value) => (value == null ? '' : String(value));
|
|
15
|
+
|
|
16
|
+
function mergeName(existing, incoming) {
|
|
17
|
+
return {
|
|
18
|
+
...existing,
|
|
19
|
+
...incoming,
|
|
20
|
+
person_id: existing.person_id,
|
|
21
|
+
source_input_id: existing.source_input_id ?? incoming.source_input_id
|
|
22
|
+
};
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
export async function transform({ batch, databaseNames = [], tablesToUpsert }) {
|
|
26
|
+
if (batch.length === 0) return;
|
|
27
|
+
const queue = [];
|
|
28
|
+
batch.forEach((o) => {
|
|
29
|
+
const present = NAME_COLUMNS.filter((column) => Object.hasOwn(o, column));
|
|
30
|
+
if (present.length === 0) return;
|
|
31
|
+
if (!o.person_id) throw new Error('Cannot update name, no person_id');
|
|
32
|
+
const row = { person_id: o.person_id, source_input_id: o.input_id ?? null };
|
|
33
|
+
for (const column of present) row[column] = o[column];
|
|
34
|
+
mergeIntoQueue(queue, row, {
|
|
35
|
+
keyFields: ['person_id'],
|
|
36
|
+
normalizeField: personKey,
|
|
37
|
+
merge: mergeName,
|
|
38
|
+
label: 'person_name'
|
|
39
|
+
});
|
|
40
|
+
});
|
|
41
|
+
if (queue.length === 0) return;
|
|
42
|
+
// Every queued row carries every column (upsertArray rejects mixed key sets),
|
|
43
|
+
// so a name the batch did not mention is filled from the stored row.
|
|
44
|
+
const stored = new Map(databaseNames.map((row) => [String(row.person_id), row]));
|
|
45
|
+
tablesToUpsert.person_name = tablesToUpsert.person_name || [];
|
|
46
|
+
for (const row of queue) {
|
|
47
|
+
const existing = stored.get(String(row.person_id));
|
|
48
|
+
const full = {
|
|
49
|
+
person_id: row.person_id,
|
|
50
|
+
source_input_id: existing?.source_input_id ?? row.source_input_id
|
|
51
|
+
};
|
|
52
|
+
for (const column of NAME_COLUMNS) {
|
|
53
|
+
full[column] = Object.hasOwn(row, column) ? row[column] : (existing?.[column] ?? null);
|
|
54
|
+
}
|
|
55
|
+
tablesToUpsert.person_name.push(full);
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
export default {
|
|
59
|
+
bindings,
|
|
60
|
+
type,
|
|
61
|
+
description,
|
|
62
|
+
transform
|
|
63
|
+
};
|
|
@@ -3,20 +3,16 @@ export const bindings = {
|
|
|
3
3
|
names: {
|
|
4
4
|
path: 'sql.query',
|
|
5
5
|
options: {
|
|
6
|
-
table: '
|
|
7
|
-
columns: ['
|
|
8
|
-
lookup: [
|
|
9
|
-
|
|
10
|
-
personIdField: 'person_id',
|
|
11
|
-
tableLookupField: 'id'
|
|
12
|
-
}
|
|
13
|
-
]
|
|
6
|
+
table: 'person_name',
|
|
7
|
+
columns: ['person_id', 'given_name', 'family_name'],
|
|
8
|
+
lookup: ['person_id'],
|
|
9
|
+
optional: true
|
|
14
10
|
}
|
|
15
11
|
}
|
|
16
12
|
};
|
|
17
13
|
export const transform = ({ batch, names }) => {
|
|
18
14
|
const nameMap = names.reduce((a, b) => {
|
|
19
|
-
a[b.
|
|
15
|
+
a[b.person_id] = a[b.person_id] || b;
|
|
20
16
|
return a;
|
|
21
17
|
}, {});
|
|
22
18
|
batch.forEach((data) => {
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
import assert from 'node:assert/strict';
|
|
2
|
+
import { describe, it } from 'node:test';
|
|
3
|
+
import upsertPersonName from './transforms/inbound/upsert_tables.js';
|
|
4
|
+
import appendName from './transforms/outbound/appendName.js';
|
|
5
|
+
import plugin from './index.js';
|
|
6
|
+
|
|
7
|
+
describe('person_name upsert_tables', () => {
|
|
8
|
+
it('is declared in the inbound upsert slot', () => {
|
|
9
|
+
assert.deepEqual(plugin.metadata.inbound, { upsert: ['upsertPersonName'] });
|
|
10
|
+
assert.equal(plugin.transforms.upsertPersonName.type, 'upsert');
|
|
11
|
+
});
|
|
12
|
+
|
|
13
|
+
it('queues one person_name row with every column', async () => {
|
|
14
|
+
const tablesToUpsert = {};
|
|
15
|
+
await upsertPersonName.transform({
|
|
16
|
+
batch: [{ person_id: 1, input_id: 'input-a', email: 'ada@example.com', given_name: 'Ada' }],
|
|
17
|
+
databaseNames: [],
|
|
18
|
+
tablesToUpsert
|
|
19
|
+
});
|
|
20
|
+
assert.deepEqual(tablesToUpsert.person_name, [
|
|
21
|
+
{ person_id: 1, source_input_id: 'input-a', given_name: 'Ada', family_name: null }
|
|
22
|
+
]);
|
|
23
|
+
assert.equal('person' in tablesToUpsert, false, 'never writes the person table');
|
|
24
|
+
});
|
|
25
|
+
|
|
26
|
+
it('keeps the stored value for a name key the row did not send', async () => {
|
|
27
|
+
const tablesToUpsert = {};
|
|
28
|
+
await upsertPersonName.transform({
|
|
29
|
+
batch: [{ person_id: 2, input_id: 'input-b', family_name: 'Lovelace' }],
|
|
30
|
+
databaseNames: [{ person_id: '2', given_name: 'Ada', family_name: 'Byron', source_input_id: 'input-a' }],
|
|
31
|
+
tablesToUpsert
|
|
32
|
+
});
|
|
33
|
+
assert.deepEqual(tablesToUpsert.person_name, [
|
|
34
|
+
{ person_id: 2, source_input_id: 'input-a', given_name: 'Ada', family_name: 'Lovelace' }
|
|
35
|
+
]);
|
|
36
|
+
});
|
|
37
|
+
|
|
38
|
+
it('clears a name when the key is present and null', async () => {
|
|
39
|
+
const tablesToUpsert = {};
|
|
40
|
+
await upsertPersonName.transform({
|
|
41
|
+
batch: [{ person_id: 3, family_name: null }],
|
|
42
|
+
databaseNames: [{ person_id: 3, given_name: 'Ada', family_name: 'Lovelace', source_input_id: null }],
|
|
43
|
+
tablesToUpsert
|
|
44
|
+
});
|
|
45
|
+
assert.equal(tablesToUpsert.person_name[0].given_name, 'Ada');
|
|
46
|
+
assert.equal(tablesToUpsert.person_name[0].family_name, null);
|
|
47
|
+
});
|
|
48
|
+
|
|
49
|
+
it('merges rows for the same person in one batch, later keys winning', async () => {
|
|
50
|
+
const tablesToUpsert = {};
|
|
51
|
+
await upsertPersonName.transform({
|
|
52
|
+
batch: [
|
|
53
|
+
{ person_id: 4, input_id: 'first', given_name: 'Ada', family_name: 'Byron' },
|
|
54
|
+
{ person_id: '4', input_id: 'second', family_name: 'Lovelace' }
|
|
55
|
+
],
|
|
56
|
+
databaseNames: [],
|
|
57
|
+
tablesToUpsert
|
|
58
|
+
});
|
|
59
|
+
assert.deepEqual(tablesToUpsert.person_name, [
|
|
60
|
+
{ person_id: 4, source_input_id: 'first', given_name: 'Ada', family_name: 'Lovelace' }
|
|
61
|
+
]);
|
|
62
|
+
});
|
|
63
|
+
|
|
64
|
+
it('queues nothing when no row has a name key', async () => {
|
|
65
|
+
const tablesToUpsert = {};
|
|
66
|
+
await upsertPersonName.transform({
|
|
67
|
+
batch: [{ person_id: 5, email: 'ada@example.com' }, { email: 'no-id@example.com' }],
|
|
68
|
+
databaseNames: [],
|
|
69
|
+
tablesToUpsert
|
|
70
|
+
});
|
|
71
|
+
assert.deepEqual(tablesToUpsert, {});
|
|
72
|
+
});
|
|
73
|
+
|
|
74
|
+
it('throws when a name is present and person_id is missing', async () => {
|
|
75
|
+
await assert.rejects(
|
|
76
|
+
() => upsertPersonName.transform({ batch: [{ given_name: 'Ada' }], databaseNames: [], tablesToUpsert: {} }),
|
|
77
|
+
/no person_id/
|
|
78
|
+
);
|
|
79
|
+
});
|
|
80
|
+
});
|
|
81
|
+
|
|
82
|
+
describe('person_name appendName', () => {
|
|
83
|
+
it('appends names by person_id', () => {
|
|
84
|
+
const batch = [{ person_id: 1 }, { person_id: 2 }];
|
|
85
|
+
appendName.transform({ batch, names: [{ person_id: 1, given_name: 'Ada', family_name: 'Lovelace' }] });
|
|
86
|
+
assert.deepEqual(batch, [
|
|
87
|
+
{ person_id: 1, given_name: 'Ada', family_name: 'Lovelace' },
|
|
88
|
+
{ person_id: 2, given_name: null, family_name: null }
|
|
89
|
+
]);
|
|
90
|
+
});
|
|
91
|
+
});
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@engine9/schemas/stacks/limited-pii",
|
|
3
|
-
"description": "PII-minimized Engine9 stack: hash identifiers instead of plaintext email, phone, or postal address. Excludes person_email, person_phone, and
|
|
4
|
-
"version": "1.
|
|
3
|
+
"description": "PII-minimized Engine9 stack: hash identifiers instead of plaintext names, email, phone, or postal address. Excludes person_name, person_email, person_phone, person_address, and transaction/contact_details even if requested.",
|
|
4
|
+
"version": "1.2.0",
|
|
5
5
|
"unique": true,
|
|
6
6
|
"include": [
|
|
7
7
|
"@engine9/schemas/plugin",
|
|
@@ -11,12 +11,13 @@
|
|
|
11
11
|
"@engine9/schemas/timeline",
|
|
12
12
|
"@engine9/schemas/source_code",
|
|
13
13
|
"@engine9/schemas/transaction/core",
|
|
14
|
-
"@engine9/schemas/transaction/contact_details",
|
|
15
14
|
"@engine9/schemas/person_hash"
|
|
16
15
|
],
|
|
17
16
|
"exclude": [
|
|
17
|
+
"@engine9/schemas/person_name",
|
|
18
18
|
"@engine9/schemas/person_email",
|
|
19
19
|
"@engine9/schemas/person_phone",
|
|
20
|
-
"@engine9/schemas/person_address"
|
|
20
|
+
"@engine9/schemas/person_address",
|
|
21
|
+
"@engine9/schemas/transaction/contact_details"
|
|
21
22
|
]
|
|
22
23
|
}
|
package/stacks/stacks.test.js
CHANGED
|
@@ -5,6 +5,8 @@ import { metadata as limitedPii } from './limited-pii/index.js';
|
|
|
5
5
|
|
|
6
6
|
test('standard stack includes plaintext contact plugins and does not exclude them', () => {
|
|
7
7
|
assert.equal(standard.name, '@engine9/schemas/stacks/standard');
|
|
8
|
+
assert.ok(standard.include.includes('@engine9/schemas/person_name'));
|
|
9
|
+
assert.ok(standard.include.includes('@engine9/schemas/transaction/contact_details'));
|
|
8
10
|
assert.ok(standard.include.includes('@engine9/schemas/person_email'));
|
|
9
11
|
assert.ok(standard.include.includes('@engine9/schemas/person_phone'));
|
|
10
12
|
assert.ok(standard.include.includes('@engine9/schemas/person_address'));
|
|
@@ -18,6 +20,10 @@ test('standard stack includes plaintext contact plugins and does not exclude the
|
|
|
18
20
|
test('limited-pii stack includes person_hash and excludes plaintext contact plugins', () => {
|
|
19
21
|
assert.equal(limitedPii.name, '@engine9/schemas/stacks/limited-pii');
|
|
20
22
|
assert.ok(limitedPii.include.includes('@engine9/schemas/person_hash'));
|
|
23
|
+
for (const path of ['@engine9/schemas/person_name', '@engine9/schemas/transaction/contact_details']) {
|
|
24
|
+
assert.ok(!limitedPii.include.includes(path), `${path} is not included`);
|
|
25
|
+
assert.ok(limitedPii.exclude.includes(path), `${path} is excluded`);
|
|
26
|
+
}
|
|
21
27
|
assert.ok(!limitedPii.include.includes('@engine9/schemas/person_email'));
|
|
22
28
|
assert.ok(!limitedPii.include.includes('@engine9/schemas/person_phone'));
|
|
23
29
|
assert.ok(!limitedPii.include.includes('@engine9/schemas/person_address'));
|
|
@@ -1,11 +1,12 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@engine9/schemas/stacks/standard",
|
|
3
|
-
"description": "Opt-in Engine9 schema stack: person, plaintext email, phone, and postal address, timeline, source codes, and transactions. Reports install separately.",
|
|
4
|
-
"version": "1.
|
|
3
|
+
"description": "Opt-in Engine9 schema stack: person, plaintext names, email, phone, and postal address, timeline, source codes, and transactions. Reports install separately.",
|
|
4
|
+
"version": "1.1.0",
|
|
5
5
|
"unique": true,
|
|
6
6
|
"include": [
|
|
7
7
|
"@engine9/schemas/plugin",
|
|
8
8
|
"@engine9/schemas/person",
|
|
9
|
+
"@engine9/schemas/person_name",
|
|
9
10
|
"@engine9/schemas/person_remote",
|
|
10
11
|
"@engine9/schemas/segment",
|
|
11
12
|
"@engine9/schemas/person_email",
|
|
@@ -1,9 +1,11 @@
|
|
|
1
|
-
// Contact details captured on a transaction: address, email, phone, employer, and occupation.
|
|
1
|
+
// Contact details captured on a transaction: payer name, address, email, phone, employer, and occupation.
|
|
2
2
|
export const tables = [
|
|
3
3
|
{
|
|
4
4
|
name: 'transaction',
|
|
5
5
|
prefix: false,
|
|
6
6
|
columns: {
|
|
7
|
+
given_name: 'string',
|
|
8
|
+
family_name: 'string',
|
|
7
9
|
street_1: 'string',
|
|
8
10
|
street_2: 'string',
|
|
9
11
|
city: 'string',
|
|
@@ -8,6 +8,8 @@ It depends on `@engine9/schemas/person` because every transaction belongs to a p
|
|
|
8
8
|
|
|
9
9
|
The plugin creates the `transaction` table. Each row is a dated amount tied to a person, an input, and optional source-code / message attribution.
|
|
10
10
|
|
|
11
|
+
The payer's name, email, phone, and address are not part of this plugin. `@engine9/schemas/transaction/contact_details` adds those columns to `transaction`; the limited-pii stack excludes it.
|
|
12
|
+
|
|
11
13
|
| Field | Type | Description |
|
|
12
14
|
| --- | --- | --- |
|
|
13
15
|
| `id` | `id_uuid` | Primary row identifier. |
|
|
@@ -22,7 +24,6 @@ The plugin creates the `transaction` table. Each row is a dated amount tied to a
|
|
|
22
24
|
| `recurs_id` | `int` | Recurring frequency. `0` is one-time. Defaults to `0`. |
|
|
23
25
|
| `recurring_number` | `int` | Sequence number in a recurring series. |
|
|
24
26
|
| `refund_amount` | `currency` | Refunded amount when applicable. |
|
|
25
|
-
| `given_name` / `family_name` / `email` | `string` | Snapshot of payer identity on the transaction. |
|
|
26
27
|
| `source_code_id` | `source_code_id` | Source code captured on the transaction. |
|
|
27
28
|
| `override_source_code_id` | `source_code_id` | Manual source-code override. |
|
|
28
29
|
| `final_source_code_id` | `source_code_id` | Source code after attribution. |
|
|
@@ -18,9 +18,6 @@ export const tables = [
|
|
|
18
18
|
},
|
|
19
19
|
recurring_number: 'int',
|
|
20
20
|
refund_amount: 'currency',
|
|
21
|
-
given_name: 'string',
|
|
22
|
-
family_name: 'string',
|
|
23
|
-
email: 'string',
|
|
24
21
|
source_code_id: 'source_code_id',
|
|
25
22
|
override_source_code_id: 'source_code_id',
|
|
26
23
|
final_source_code_id: 'source_code_id',
|
|
@@ -4,7 +4,7 @@ export const settings = [
|
|
|
4
4
|
type: 'boolean',
|
|
5
5
|
default: false,
|
|
6
6
|
description:
|
|
7
|
-
'When true, installDefaultPlugins prefers @engine9/schemas/stacks/limited-pii over default_stack, and installs of stacks/standard, person_email, person_phone, and
|
|
7
|
+
'When true, installDefaultPlugins prefers @engine9/schemas/stacks/limited-pii over default_stack, and installs of stacks/standard, person_name, person_email, person_phone, person_address, and transaction/contact_details are refused. Does not drop existing plaintext tables. Also omits the plaintext person tables from the default export list when tables is not set.'
|
|
8
8
|
}
|
|
9
9
|
];
|
|
10
10
|
export default {
|