@engine9/schemas 1.9.1 → 1.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -3
- package/channels/email/README.md +4 -4
- package/channels/email/index.js +3 -3
- package/channels/email/{search.js → segmentDefinitionClauses.js} +3 -2
- package/channels/email/segments.js +7 -7
- package/event/README.md +2 -2
- package/event/index.js +3 -3
- package/event/{search.js → segmentDefinitionClauses.js} +2 -1
- package/package.json +2 -2
- package/person/index.js +10 -25
- package/person/schema.js +0 -2
- package/person/ui.console.json5 +14 -71
- package/person_email/README.md +3 -3
- package/person_email/index.js +3 -3
- package/person_email/{search.js → segmentDefinitionClauses.js} +2 -1
- package/person_email/segments.js +2 -2
- package/person_hash/README.md +4 -4
- package/person_hash/index.js +3 -3
- package/person_hash/{search.js → segmentDefinitionClauses.js} +2 -2
- package/person_name/README.md +69 -0
- package/person_name/index.js +28 -0
- package/person_name/schema.js +17 -0
- package/person_name/segmentDefinitionClauses.js +48 -0
- package/person_name/transforms/inbound/upsert_tables.js +63 -0
- package/{person → person_name}/transforms/outbound/appendName.js +5 -9
- package/person_name/upsert.test.js +91 -0
- package/person_phone/README.md +2 -2
- package/person_phone/index.js +3 -3
- package/person_phone/{search.js → segmentDefinitionClauses.js} +2 -1
- package/person_phone/segments.js +1 -1
- package/person_remote/index.js +4 -5
- package/segment/index.js +3 -3
- package/segment/schema.js +3 -3
- package/segment/{search.js → segmentDefinitionClauses.js} +2 -1
- package/stacks/limited-pii/stack.json +5 -4
- package/stacks/stacks.test.js +6 -0
- package/stacks/standard/stack.json +3 -2
- package/transaction/contact_details/schema.js +3 -1
- package/transaction/core/README.md +7 -6
- package/transaction/core/index.js +3 -3
- package/transaction/core/schema.js +0 -3
- package/transaction/core/{search.js → segmentDefinitionClauses.js} +2 -2
- package/transaction/core/segments.js +1 -1
- package/utilities/limited-pii/settings.js +1 -1
package/README.md
CHANGED
|
@@ -7,7 +7,7 @@ this code as-is.
|
|
|
7
7
|
|
|
8
8
|
## Documentation
|
|
9
9
|
|
|
10
|
-
Each schema plugin is documented in its own `README.md`. That file is the human-readable contract: what the schema plugin is for, the data model, inbound/outbound behavior,
|
|
10
|
+
Each schema plugin is documented in its own `README.md`. That file is the human-readable contract: what the schema plugin is for, the data model, inbound/outbound behavior, segment definition clauses, **predefined segments**, and UI. Keep implementation details in code comments; keep audience and membership rules in the README. Dashboards live on `@engine9/plugins/reports/<area>`, not in this package.
|
|
11
11
|
|
|
12
12
|
Schema plugins with predefined segments:
|
|
13
13
|
|
|
@@ -22,9 +22,9 @@ See [`skills/create-engine9-plugin/SKILL.md`](../skills/create-engine9-plugin/SK
|
|
|
22
22
|
|
|
23
23
|
- **Plugin uniqueness:** `SchemaWorker.install` reuses an existing `plugin` row when the path is unique: `metadata.unique` if set, otherwise `@engine9/schemas/*` (except `person_custom`, which is `unique: false`). Native plugins that set `unique: true` behave the same. Third-party plugins and `person_custom` may have more than one instance per path; pass `id` to update an existing non-unique row.
|
|
24
24
|
|
|
25
|
-
- **Public module surface:** Export only the standard building blocks documented in [`skills/create-engine9-plugin/SKILL.md`](../skills/create-engine9-plugin/SKILL.md): for example `metadata`, optional `schema`, `transforms`, `
|
|
25
|
+
- **Public module surface:** Export only the standard building blocks documented in [`skills/create-engine9-plugin/SKILL.md`](../skills/create-engine9-plugin/SKILL.md): for example `metadata`, optional `schema`, `transforms`, `segmentDefinitionClauses`, `segments`, `metrics`, and the default object that aggregates them. Do **not** export `reports` from schema plugins — dashboards belong on `@engine9/plugins/reports/<area>`. Do **not** export ad hoc hooks (such as custom install helpers or segment–plugin wiring) from schema plugins; the server is responsible for any special install-time behavior tied to a specific schema plugin path.
|
|
26
26
|
|
|
27
|
-
- **Segments, universes, and `plugin_id`:** Saved segments use `segment.plugin_id` = the schema plugin (or plugin) that **owns the definition**. A definition’s **`universe`** describes the span of source data searched across; input EQL entries select timeline stores, while person/table entries can restrict eligible people. Explicit `pluginId` values inside
|
|
27
|
+
- **Segments, universes, and `plugin_id`:** Saved segments use `segment.plugin_id` = the schema plugin (or plugin) that **owns the definition**. A definition’s **`universe`** describes the span of source data searched across; input EQL entries select timeline stores, while person/table entries can restrict eligible people. Explicit `pluginId` values inside a clause's options may narrow that span further, so a segment can evaluate data tied to other deployed plugins without changing its owning `plugin_id`. Export bundles use the same universe vocabulary for table and input-file artifacts.
|
|
28
28
|
|
|
29
29
|
## License
|
|
30
30
|
|
package/channels/email/README.md
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
# Email Channel Schema Plugin
|
|
2
2
|
|
|
3
|
-
The `@engine9/schemas/channels/email` plugin defines email-channel
|
|
3
|
+
The `@engine9/schemas/channels/email` plugin defines the email-channel segment definition clause and engagement segments. It does not create its own tables. Segment builds read `global_message`, `input`, and `timeline` rows produced by email message plugins.
|
|
4
4
|
|
|
5
5
|
It depends on `@engine9/schemas/person` because membership is always a set of people.
|
|
6
6
|
|
|
7
|
-
##
|
|
7
|
+
## Segment definition clauses
|
|
8
8
|
|
|
9
9
|
`emailEngagement` finds people who opened or clicked email in a rolling window.
|
|
10
10
|
|
|
@@ -14,13 +14,13 @@ It depends on `@engine9/schemas/person` because membership is always a set of pe
|
|
|
14
14
|
| `timelineEntryType` | `EMAIL_OPEN` or `EMAIL_CLICK` |
|
|
15
15
|
| `pluginId` | Optional. When set, only that plugin's inputs are included. When empty, scope comes from the segment universe instead |
|
|
16
16
|
|
|
17
|
-
The
|
|
17
|
+
The clause looks at `timeline` rows whose `entry_type_id` matches the engagement type and whose `ts` is in the window. It always joins `timeline.input_id = input.id`, so only timeline entries for inputs present in the build are counted.
|
|
18
18
|
|
|
19
19
|
## Segments
|
|
20
20
|
|
|
21
21
|
Predefined audiences shipped with this schema plugin. On install, each key becomes a `segment` row whose definition path is `@engine9/schemas/channels/email:segments:<key>`.
|
|
22
22
|
|
|
23
|
-
Every segment here uses the same **universe** and the `emailEngagement`
|
|
23
|
+
Every segment here uses the same **universe** and the `emailEngagement` clause. The universe and the clause do different jobs, and both are required:
|
|
24
24
|
|
|
25
25
|
- **Universe** chooses *which email sends* can contribute entries: messages on the email channel published in the last **90 days** (`global_message.channel = 'email'` and `publish_date` within 90 days). Those message ids become the `input` rows for the build.
|
|
26
26
|
- **Search** chooses *which people* from those sends belong in the audience: a recent open or click on the timeline, inside the segment's rolling window.
|
package/channels/email/index.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import
|
|
1
|
+
import { segmentDefinitionClauses } from './segmentDefinitionClauses.js';
|
|
2
2
|
import segments, { personSegmentTableName, universeEmailPublished90d } from './segments.js';
|
|
3
3
|
|
|
4
4
|
const metadata = {
|
|
@@ -9,13 +9,13 @@ const metadata = {
|
|
|
9
9
|
};
|
|
10
10
|
|
|
11
11
|
export { metadata };
|
|
12
|
-
export {
|
|
12
|
+
export { segmentDefinitionClauses };
|
|
13
13
|
export { segments };
|
|
14
14
|
export { personSegmentTableName };
|
|
15
15
|
export { universeEmailPublished90d };
|
|
16
16
|
export default {
|
|
17
17
|
metadata,
|
|
18
|
-
|
|
18
|
+
segmentDefinitionClauses,
|
|
19
19
|
segments,
|
|
20
20
|
personSegmentTableName,
|
|
21
21
|
universeEmailPublished90d
|
|
@@ -3,7 +3,7 @@ import { TIMELINE_ENTRY_TYPES } from '@engine9/input-tools';
|
|
|
3
3
|
/**
|
|
4
4
|
* Build EQL for email engagement (opens / clicks) within a rolling window.
|
|
5
5
|
*
|
|
6
|
-
* The
|
|
6
|
+
* The clause always joins `timeline` → `input` on `timeline.input_id = input.id`.
|
|
7
7
|
* During a segment build the DuckDB `input` table is populated exclusively with
|
|
8
8
|
* the input ids discovered by the segment's `universe` entry (e.g.
|
|
9
9
|
* `universeEmailPublished90d`), so the join scopes results to the universe even
|
|
@@ -81,6 +81,7 @@ export const emailEngagement = {
|
|
|
81
81
|
}
|
|
82
82
|
};
|
|
83
83
|
|
|
84
|
+
export const segmentDefinitionClauses = { emailEngagement };
|
|
84
85
|
export default {
|
|
85
|
-
|
|
86
|
+
segmentDefinitionClauses
|
|
86
87
|
};
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
const
|
|
1
|
+
const EMAIL_ENGAGEMENT_CLAUSE_PATH = '@engine9/schemas/channels/email:segment_definition_clause:emailEngagement';
|
|
2
2
|
|
|
3
3
|
/**
|
|
4
4
|
* EQL over the message table: rows published in the last 90 days on the email channel.
|
|
@@ -6,12 +6,12 @@ const SEGMENT_SEARCH_PATH = '@engine9/schemas/channels/email:search:emailEngagem
|
|
|
6
6
|
* scanned for timeline parquet.
|
|
7
7
|
*
|
|
8
8
|
* During a segment build the input ids returned here are copied into the DuckDB `input`
|
|
9
|
-
* table. The
|
|
9
|
+
* table. The clause EQL (see `emailEngagement` in `segmentDefinitionClauses.js`) always joins
|
|
10
10
|
* `timeline.input_id = input.id`, so only timeline rows whose input is in **this**
|
|
11
11
|
* universe are considered — even when `pluginId` is empty. This means:
|
|
12
12
|
*
|
|
13
13
|
* - The **universe** controls *which* message/input stores supply timeline data.
|
|
14
|
-
* - The **
|
|
14
|
+
* - The **clause** applies time-window and entry-type filters *within* that scope.
|
|
15
15
|
*
|
|
16
16
|
* Both layers are required for correct segment membership counts.
|
|
17
17
|
*/
|
|
@@ -35,7 +35,7 @@ export function personSegmentTableName(tablePrefix, segmentName) {
|
|
|
35
35
|
/**
|
|
36
36
|
* Build a segment definition for email engagement (opens or clicks).
|
|
37
37
|
*
|
|
38
|
-
* `pluginId` is left empty intentionally: the
|
|
38
|
+
* `pluginId` is left empty intentionally: the `emailEngagement` clause
|
|
39
39
|
* always joins `timeline` → `input`, and the segment `universe` determines
|
|
40
40
|
* which input rows are present in DuckDB. This scopes the time-window query
|
|
41
41
|
* to the correct set of message inputs without hard-coding a plugin id.
|
|
@@ -44,10 +44,10 @@ function emailEngagementSegment(name, windowDays, timelineEntryType) {
|
|
|
44
44
|
return {
|
|
45
45
|
name,
|
|
46
46
|
universe: [universeEmailPublished90d],
|
|
47
|
-
|
|
47
|
+
definition: {
|
|
48
48
|
and: [
|
|
49
49
|
{
|
|
50
|
-
path:
|
|
50
|
+
path: EMAIL_ENGAGEMENT_CLAUSE_PATH,
|
|
51
51
|
options: {
|
|
52
52
|
pluginId: '',
|
|
53
53
|
windowDays,
|
|
@@ -61,7 +61,7 @@ function emailEngagementSegment(name, windowDays, timelineEntryType) {
|
|
|
61
61
|
|
|
62
62
|
/**
|
|
63
63
|
* Rolling-window engagement segments. `pluginId` is empty so scope follows the
|
|
64
|
-
* segment `universe` (the `input` table join in the
|
|
64
|
+
* segment `universe` (the `input` table join in the clause EQL restricts to
|
|
65
65
|
* universe-provided message input ids).
|
|
66
66
|
*/
|
|
67
67
|
export const email_openers_30d = emailEngagementSegment('30-day email openers', 30, 'EMAIL_OPEN');
|
package/event/README.md
CHANGED
|
@@ -56,11 +56,11 @@ None in this package. Native event / calendar plugins should upsert `event` and
|
|
|
56
56
|
|
|
57
57
|
None.
|
|
58
58
|
|
|
59
|
-
##
|
|
59
|
+
## Segment definition clauses
|
|
60
60
|
|
|
61
61
|
| Key | Path | Purpose |
|
|
62
62
|
| --- | --- | --- |
|
|
63
|
-
| `responses` | `@engine9/schemas/event:
|
|
63
|
+
| `responses` | `@engine9/schemas/event:segment_definition_clause:responses` | Filter people by `person_event.response` and optional `eventId`. |
|
|
64
64
|
|
|
65
65
|
## Segments
|
|
66
66
|
|
package/event/index.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import schema from './schema.js';
|
|
2
|
-
import
|
|
2
|
+
import { segmentDefinitionClauses } from './segmentDefinitionClauses.js';
|
|
3
3
|
|
|
4
4
|
const metadata = {
|
|
5
5
|
name: '@engine9/schemas/event',
|
|
@@ -9,9 +9,9 @@ const metadata = {
|
|
|
9
9
|
schemas: ['schema.js']
|
|
10
10
|
};
|
|
11
11
|
|
|
12
|
-
export { metadata, schema,
|
|
12
|
+
export { metadata, schema, segmentDefinitionClauses };
|
|
13
13
|
export default {
|
|
14
14
|
metadata,
|
|
15
15
|
schema,
|
|
16
|
-
|
|
16
|
+
segmentDefinitionClauses
|
|
17
17
|
};
|
package/package.json
CHANGED
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@engine9/schemas",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.11.0",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"description": "Engine9 standard schemas: shared tables, transforms, searches, and segments",
|
|
6
6
|
"main": "index.js",
|
|
7
7
|
"scripts": {
|
|
8
|
-
"test": "node --test person/upsert.test.js person_address/normalize.test.js person_email/upsert.test.js person_hash/hash.test.js person_phone/upsert.test.js stacks/stacks.test.js transaction/core/attribute.test.js"
|
|
8
|
+
"test": "node --test person/upsert.test.js person_address/normalize.test.js person_email/upsert.test.js person_hash/hash.test.js person_name/upsert.test.js person_phone/upsert.test.js stacks/stacks.test.js transaction/core/attribute.test.js"
|
|
9
9
|
},
|
|
10
10
|
"engines": {
|
|
11
11
|
"node": ">=18"
|
package/person/index.js
CHANGED
|
@@ -2,38 +2,31 @@ import schema from './schema.js';
|
|
|
2
2
|
import upsertPerson from './transforms/inbound/upsert_tables.js';
|
|
3
3
|
import normalizeFieldNames from './transforms/inbound/normalize_field_names.js';
|
|
4
4
|
import simpleMap from './transforms/simpleMap.js';
|
|
5
|
-
import appendName from '
|
|
5
|
+
import appendName from '../person_name/transforms/outbound/appendName.js';
|
|
6
6
|
import metrics from './metrics.js';
|
|
7
7
|
const metadata = {
|
|
8
8
|
name: '@engine9/schemas/person',
|
|
9
9
|
schemas: ['schema.js'],
|
|
10
10
|
// Inbound people pipeline slots -> transform export keys (woven by core when installed)
|
|
11
11
|
inbound: {
|
|
12
|
-
normalize: ['normalizeFieldNames']
|
|
13
|
-
upsert: ['upsertPerson']
|
|
12
|
+
normalize: ['normalizeFieldNames']
|
|
14
13
|
}
|
|
15
14
|
};
|
|
16
15
|
const transforms = {
|
|
17
16
|
normalizeFieldNames,
|
|
17
|
+
// Not woven. Plugin rows installed before person_name still name it in their
|
|
18
|
+
// inbound snapshot until the person_name upgrade refreshes that snapshot.
|
|
18
19
|
upsertPerson,
|
|
19
20
|
simpleMap,
|
|
21
|
+
// Saved export definitions still call @engine9/schemas/person:transforms:appendName.
|
|
20
22
|
appendName
|
|
21
23
|
};
|
|
22
|
-
|
|
23
|
-
person: {
|
|
24
|
+
const person = {
|
|
24
25
|
title: 'Primary Info',
|
|
25
26
|
form: {
|
|
26
27
|
title: 'Primary Info',
|
|
27
28
|
type: 'object',
|
|
28
29
|
properties: {
|
|
29
|
-
givenName: {
|
|
30
|
-
title: 'First Name',
|
|
31
|
-
type: 'string'
|
|
32
|
-
},
|
|
33
|
-
familyName: {
|
|
34
|
-
title: 'Last Name',
|
|
35
|
-
type: 'string'
|
|
36
|
-
},
|
|
37
30
|
ids: {
|
|
38
31
|
title: 'Person Ids (comma delimited)',
|
|
39
32
|
type: 'string'
|
|
@@ -51,7 +44,7 @@ export const search = {
|
|
|
51
44
|
},
|
|
52
45
|
optionsToEQL: (options) => {
|
|
53
46
|
const text = [];
|
|
54
|
-
const { ids, idGreaterThanEqual, idLessThan
|
|
47
|
+
const { ids, idGreaterThanEqual, idLessThan } = options;
|
|
55
48
|
const conditions = [];
|
|
56
49
|
if (ids) {
|
|
57
50
|
let arr = ids || '';
|
|
@@ -68,14 +61,6 @@ export const search = {
|
|
|
68
61
|
conditions.push({ eql: `id<${idLessThan}` });
|
|
69
62
|
text.push(`Person ID less than ${idLessThan}`);
|
|
70
63
|
}
|
|
71
|
-
if (givenName.length > 0) {
|
|
72
|
-
text.push(`First name like '${givenName}'`);
|
|
73
|
-
conditions.push({ eql: `given_name like '${givenName}'` });
|
|
74
|
-
}
|
|
75
|
-
if (familyName.length > 0) {
|
|
76
|
-
text.push(`Last name like '${givenName}'`);
|
|
77
|
-
conditions.push({ eql: `family_name like '${familyName}'` });
|
|
78
|
-
}
|
|
79
64
|
return {
|
|
80
65
|
text,
|
|
81
66
|
eql: {
|
|
@@ -85,8 +70,8 @@ export const search = {
|
|
|
85
70
|
}
|
|
86
71
|
};
|
|
87
72
|
}
|
|
88
|
-
}
|
|
89
|
-
};
|
|
73
|
+
};
|
|
74
|
+
export const segmentDefinitionClauses = { person };
|
|
90
75
|
export { metadata };
|
|
91
76
|
export { schema };
|
|
92
77
|
export { transforms };
|
|
@@ -96,5 +81,5 @@ export default {
|
|
|
96
81
|
schema,
|
|
97
82
|
transforms,
|
|
98
83
|
metrics,
|
|
99
|
-
|
|
84
|
+
segmentDefinitionClauses
|
|
100
85
|
};
|
package/person/schema.js
CHANGED
package/person/ui.console.json5
CHANGED
|
@@ -17,26 +17,16 @@
|
|
|
17
17
|
person: {
|
|
18
18
|
layout: 'full_width',
|
|
19
19
|
components: {
|
|
20
|
-
header: [
|
|
21
|
-
'Everybody',
|
|
22
|
-
{
|
|
23
|
-
component: 'Button',
|
|
24
|
-
properties: {
|
|
25
|
-
icon: 'plus',
|
|
26
|
-
content: 'Add Person',
|
|
27
|
-
onClick: {
|
|
28
|
-
action: 'navigate',
|
|
29
|
-
url: '/person/create'
|
|
30
|
-
}
|
|
31
|
-
}
|
|
32
|
-
}
|
|
33
|
-
],
|
|
20
|
+
header: ['Everybody'],
|
|
34
21
|
main: [
|
|
35
22
|
{
|
|
36
23
|
component: 'RecordList',
|
|
37
24
|
properties: {
|
|
38
25
|
table: 'person',
|
|
39
26
|
includes: {
|
|
27
|
+
names: {
|
|
28
|
+
table: 'person_name'
|
|
29
|
+
},
|
|
40
30
|
emails: {
|
|
41
31
|
table: 'person_email'
|
|
42
32
|
}
|
|
@@ -51,8 +41,7 @@
|
|
|
51
41
|
{
|
|
52
42
|
title: 'Name',
|
|
53
43
|
dataIndex: 'name',
|
|
54
|
-
|
|
55
|
-
template: '{{given_name}} {{family_name}} {{emails.0.email}}',
|
|
44
|
+
template: '{{names.0.given_name}} {{names.0.family_name}} {{emails.0.email}}',
|
|
56
45
|
width: '40%'
|
|
57
46
|
},
|
|
58
47
|
{
|
|
@@ -72,36 +61,6 @@
|
|
|
72
61
|
]
|
|
73
62
|
}
|
|
74
63
|
},
|
|
75
|
-
'person/create': {
|
|
76
|
-
layout: 'full_width',
|
|
77
|
-
components: {
|
|
78
|
-
main: [
|
|
79
|
-
{
|
|
80
|
-
path: 'edit',
|
|
81
|
-
component: 'RecordForm',
|
|
82
|
-
properties: {
|
|
83
|
-
table: 'person',
|
|
84
|
-
form: {
|
|
85
|
-
title: 'Todo',
|
|
86
|
-
type: 'object',
|
|
87
|
-
required: [],
|
|
88
|
-
properties: {
|
|
89
|
-
given_name: { type: 'string', title: 'First Name', default: '' },
|
|
90
|
-
family_name: { type: 'string', title: 'Last Name', default: '' }
|
|
91
|
-
}
|
|
92
|
-
},
|
|
93
|
-
uiSchema: {
|
|
94
|
-
family_name: { 'ui:widget': 'textarea' }
|
|
95
|
-
},
|
|
96
|
-
onSave: {
|
|
97
|
-
action: 'navigate',
|
|
98
|
-
url: '/person/{{record.id}}'
|
|
99
|
-
}
|
|
100
|
-
}
|
|
101
|
-
}
|
|
102
|
-
]
|
|
103
|
-
}
|
|
104
|
-
},
|
|
105
64
|
'person/:id/*': {
|
|
106
65
|
layout: 'sidebar',
|
|
107
66
|
components: {
|
|
@@ -114,19 +73,15 @@
|
|
|
114
73
|
table: 'person',
|
|
115
74
|
id: '{{parameters.id}}',
|
|
116
75
|
includes: {
|
|
76
|
+
names: {
|
|
77
|
+
table: 'person_name'
|
|
78
|
+
},
|
|
117
79
|
emails: {
|
|
118
80
|
table: 'person_email',
|
|
119
81
|
orderBy: 'preference_order'
|
|
120
82
|
}
|
|
121
83
|
},
|
|
122
|
-
template: '{{given_name}} {{family_name}} {{#if emails.0.email}}<{{emails.0.email}}>{{/if}}'
|
|
123
|
-
}
|
|
124
|
-
},
|
|
125
|
-
{
|
|
126
|
-
component: 'Link',
|
|
127
|
-
properties: {
|
|
128
|
-
label: 'Edit',
|
|
129
|
-
to: 'edit'
|
|
84
|
+
template: '{{names.0.given_name}} {{names.0.family_name}} {{#if emails.0.email}}<{{emails.0.email}}>{{/if}}'
|
|
130
85
|
}
|
|
131
86
|
}
|
|
132
87
|
]
|
|
@@ -138,24 +93,12 @@
|
|
|
138
93
|
properties: {
|
|
139
94
|
table: 'person',
|
|
140
95
|
id: '{{parameters.id}}',
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
{
|
|
145
|
-
path: 'edit',
|
|
146
|
-
component: 'RecordForm',
|
|
147
|
-
properties: {
|
|
148
|
-
table: 'person',
|
|
149
|
-
id: '{{parameters.id}}',
|
|
150
|
-
form: {
|
|
151
|
-
title: 'Todo',
|
|
152
|
-
type: 'object',
|
|
153
|
-
required: [],
|
|
154
|
-
properties: {
|
|
155
|
-
given_name: { type: 'string', title: 'First Name', default: '' },
|
|
156
|
-
family_name: { type: 'string', title: 'Last Name', default: '' }
|
|
96
|
+
includes: {
|
|
97
|
+
names: {
|
|
98
|
+
table: 'person_name'
|
|
157
99
|
}
|
|
158
|
-
}
|
|
100
|
+
},
|
|
101
|
+
template: '{{names.0.given_name}} {{names.0.family_name}}'
|
|
159
102
|
}
|
|
160
103
|
}
|
|
161
104
|
]
|
package/person_email/README.md
CHANGED
|
@@ -51,9 +51,9 @@ The plugin provides outbound transforms for appending email data to batches:
|
|
|
51
51
|
- `appendEmail` looks up `person_email` rows by `person_id` and appends `email` to each batch row. It can be filtered by `subscriptionStatus`, and it preserves any existing `email` value already on the row.
|
|
52
52
|
- `appendEmailHash` appends `email_hash_v1` by hashing the row's trimmed, lowercased `email` when the hash is not already present.
|
|
53
53
|
|
|
54
|
-
##
|
|
54
|
+
## Segment definition clauses
|
|
55
55
|
|
|
56
|
-
The `emails`
|
|
56
|
+
The `emails` clause selects people who have an email address, optionally filtered by:
|
|
57
57
|
|
|
58
58
|
- `subscriptionStatus`, using the same values as `subscription_status`
|
|
59
59
|
- `emailMatch`, matched against the `email` column with `LIKE`
|
|
@@ -70,7 +70,7 @@ Predefined audiences shipped with this schema plugin. On install, each key becom
|
|
|
70
70
|
| **Definition path** | `@engine9/schemas/person_email:segments:subscribers` |
|
|
71
71
|
| **Who is included** | People with at least one email whose `subscription_status` is `Subscribed` |
|
|
72
72
|
| **Who is excluded** | People whose emails are only `Not Subscribed`, `Unsubscribed`, `Bouncing`, or `Spam` |
|
|
73
|
-
| **How it is built** | The `emails`
|
|
73
|
+
| **How it is built** | The `emails` clause with `subscriptionStatus: Subscribed` |
|
|
74
74
|
| **Universe** | None — membership is current `person_email` rows, not a time window or message input set |
|
|
75
75
|
|
|
76
76
|
A person with mixed statuses is included if any of their addresses is `Subscribed`.
|
package/person_email/index.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import schema from './schema.js';
|
|
2
2
|
import extractEmailHashes from './transforms/inbound/extract_identifiers.js';
|
|
3
3
|
import upsertPersonEmail from './transforms/inbound/upsert_tables.js';
|
|
4
|
-
import
|
|
4
|
+
import { segmentDefinitionClauses } from './segmentDefinitionClauses.js';
|
|
5
5
|
import appendEmail from './transforms/outbound/appendEmail.js';
|
|
6
6
|
import appendEmailHash from './transforms/outbound/appendEmailHash.js';
|
|
7
7
|
import segments from './segments.js';
|
|
@@ -24,12 +24,12 @@ export const transforms = {
|
|
|
24
24
|
};
|
|
25
25
|
export { metadata };
|
|
26
26
|
export { schema };
|
|
27
|
-
export {
|
|
27
|
+
export { segmentDefinitionClauses };
|
|
28
28
|
export { segments };
|
|
29
29
|
export default {
|
|
30
30
|
metadata,
|
|
31
31
|
schema,
|
|
32
|
-
|
|
32
|
+
segmentDefinitionClauses,
|
|
33
33
|
segments,
|
|
34
34
|
transforms
|
|
35
35
|
};
|
package/person_email/segments.js
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
export const subscribers = {
|
|
2
2
|
name: 'Email Subscribers',
|
|
3
|
-
|
|
3
|
+
definition: {
|
|
4
4
|
and: [
|
|
5
5
|
{
|
|
6
|
-
path: '@engine9/schemas/person_email:
|
|
6
|
+
path: '@engine9/schemas/person_email:segment_definition_clause:emails',
|
|
7
7
|
options: {
|
|
8
8
|
subscriptionStatus: 'Subscribed'
|
|
9
9
|
}
|
package/person_hash/README.md
CHANGED
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
|
|
5
5
|
It is **not** part of the standard stack. It ships in `@engine9/schemas/stacks/limited-pii`, or install it explicitly (once per account — `unique: true`). Once installed, core weaves its transforms into the inbound people pipeline automatically.
|
|
6
6
|
|
|
7
|
-
Warehouse plugin setting `exclude_pii` on `@engine9/schemas/utilities/limited-pii` makes `installDefaultPlugins()` prefer the limited-pii stack over `default_stack` and refuses `@engine9/schemas/stacks/standard`, `person_email`, `person_phone`, and `
|
|
7
|
+
Warehouse plugin setting `exclude_pii` on `@engine9/schemas/utilities/limited-pii` makes `installDefaultPlugins()` prefer the limited-pii stack over `default_stack` and refuses `@engine9/schemas/stacks/standard`, `person_name`, `person_email`, `person_phone`, `person_address`, and `transaction/contact_details` even if those plugins are already installed. That does **not** uninstall leftover plaintext tables. Install the utility plugin on demand; it is not part of the limited-pii stack.
|
|
8
8
|
|
|
9
9
|
It does **not** depend on `person_email` or `person_phone`. Those plugins may be installed alongside it; this schema plugin never writes to their tables.
|
|
10
10
|
|
|
@@ -31,7 +31,7 @@ inbound: {
|
|
|
31
31
|
|
|
32
32
|
How slots and the weaver work: `@engine9/core/lib/peoplePipeline/README.md`.
|
|
33
33
|
|
|
34
|
-
To see the woven chain for an account: `personWorker.getInboundTransforms({ pluginId, describe: true })` (CLI: `e9 personworker getInboundTransforms --plugin_id=<plugin_id> --describe=true`). On a limited-pii account the listing has `person_hash` on `id` and `upsert` and no `person_email` / `person_phone` / `person_address` lines. Drop a step for one job with `omit_transforms`.
|
|
34
|
+
To see the woven chain for an account: `personWorker.getInboundTransforms({ pluginId, describe: true })` (CLI: `e9 personworker getInboundTransforms --plugin_id=<plugin_id> --describe=true`). On a limited-pii account the listing has `person_hash` on `id` and `upsert` and no `person_name` / `person_email` / `person_phone` / `person_address` lines. Drop a step for one job with `omit_transforms`.
|
|
35
35
|
|
|
36
36
|
## Data Model
|
|
37
37
|
|
|
@@ -65,6 +65,6 @@ The `upsertPersonHash` transform writes only hash columns to `person_hash_email`
|
|
|
65
65
|
|
|
66
66
|
`PersonWorker.search` `emails` / `phones` match `person_email` / `person_phone` when those tables exist, and also hash the query (or accept SHA-256 / MD5 hex) against `person_hash_email` / `person_hash_phone`. If the caller passed emails or phones and neither table exists, the clause matches nobody (`1=0`). Related summaries include `email_hashes` / `phone_hashes` when those tables are present.
|
|
67
67
|
|
|
68
|
-
The plugin
|
|
68
|
+
The plugin exports `segmentDefinitionClauses.emailHashes` and `segmentDefinitionClauses.phoneHashes` for segment definitions (`@engine9/schemas/person_hash:segment_definition_clause:emailHashes`).
|
|
69
69
|
|
|
70
|
-
Default warehouse export includes the hash tables (export skips them when they do not exist). When utilities/limited-pii `exclude_pii` is set and `tables` is not explicit, `person_email` / `person_phone` / `person_address` are omitted from that default list.
|
|
70
|
+
Default warehouse export includes the hash tables (export skips them when they do not exist). When utilities/limited-pii `exclude_pii` is set and `tables` is not explicit, `person_name` / `person_email` / `person_phone` / `person_address` are omitted from that default list.
|
package/person_hash/index.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import schema from './schema.js';
|
|
2
|
-
import
|
|
2
|
+
import { segmentDefinitionClauses } from './segmentDefinitionClauses.js';
|
|
3
3
|
import extractContactHashes from './transforms/inbound/extract_identifiers.js';
|
|
4
4
|
import upsertPersonHash from './transforms/inbound/upsert_tables.js';
|
|
5
5
|
import appendEmailHash from './transforms/outbound/appendEmailHash.js';
|
|
@@ -28,10 +28,10 @@ export const transforms = {
|
|
|
28
28
|
|
|
29
29
|
export { metadata };
|
|
30
30
|
export { schema };
|
|
31
|
-
export {
|
|
31
|
+
export { segmentDefinitionClauses };
|
|
32
32
|
export default {
|
|
33
33
|
metadata,
|
|
34
34
|
schema,
|
|
35
35
|
transforms,
|
|
36
|
-
|
|
36
|
+
segmentDefinitionClauses
|
|
37
37
|
};
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
# Person Name Schema Plugin
|
|
2
|
+
|
|
3
|
+
`@engine9/schemas/person_name` stores a person's plaintext name: `given_name` and `family_name`, one row per person.
|
|
4
|
+
|
|
5
|
+
Names are personal data, so they are not on `person`. `person` holds only the id and timestamps, and every account installs it. An account that must not store names leaves this plugin out, the same way it leaves out `person_email`, `person_phone`, and `person_address`.
|
|
6
|
+
|
|
7
|
+
- The standard stack (`@engine9/schemas/stacks/standard`) and the no-argument `installDefaultPlugins()` install it.
|
|
8
|
+
- The limited-pii stack (`@engine9/schemas/stacks/limited-pii`) excludes it. With `exclude_pii` set on `@engine9/schemas/utilities/limited-pii`, core refuses to install it.
|
|
9
|
+
|
|
10
|
+
## Install
|
|
11
|
+
|
|
12
|
+
```javascript
|
|
13
|
+
await pluginWorker.install({ path: '@engine9/schemas/person_name' });
|
|
14
|
+
```
|
|
15
|
+
|
|
16
|
+
A second install of the same path reuses the existing plugin row.
|
|
17
|
+
|
|
18
|
+
## Inbound pipeline slots
|
|
19
|
+
|
|
20
|
+
```javascript
|
|
21
|
+
inbound: {
|
|
22
|
+
upsert: ['upsertPersonName']
|
|
23
|
+
}
|
|
24
|
+
```
|
|
25
|
+
|
|
26
|
+
When the plugin is installed, core weaves `upsertPersonName` into the `upsert` slot of the inbound people pipeline. When it is not installed, the step is not in the chain. Incoming `given_name` / `family_name` fields are then ignored and nothing is stored. How slots and the weaver work: `@engine9/core/lib/peoplePipeline/README.md`.
|
|
27
|
+
|
|
28
|
+
## Data Model
|
|
29
|
+
|
|
30
|
+
### `person_name`
|
|
31
|
+
|
|
32
|
+
Primary key: `person_id`.
|
|
33
|
+
|
|
34
|
+
| Field | Type | Description |
|
|
35
|
+
| --- | --- | --- |
|
|
36
|
+
| `person_id` | `person_id` | Person the name belongs to. |
|
|
37
|
+
| `given_name` | `string` | Given (first) name. |
|
|
38
|
+
| `family_name` | `string` | Family (last) name. |
|
|
39
|
+
| `source_input_id` | `foreign_uuid` | Input that first wrote the row. Existing rows keep their original source. Rows copied by the core `person_name` upgrade have none. |
|
|
40
|
+
| `created_at` / `modified_at` | timestamps | |
|
|
41
|
+
|
|
42
|
+
## Inbound Behavior
|
|
43
|
+
|
|
44
|
+
`upsertPersonName` runs for each row that has a `given_name` or `family_name` key:
|
|
45
|
+
|
|
46
|
+
- A key that is present overwrites the stored value. An explicit `null` clears it.
|
|
47
|
+
- A key that is absent keeps the stored value.
|
|
48
|
+
- Rows for the same person in one batch merge; later keys win.
|
|
49
|
+
- A row with a name and no `person_id` fails the batch.
|
|
50
|
+
|
|
51
|
+
## Outbound Behavior
|
|
52
|
+
|
|
53
|
+
`appendName` adds `given_name` and `family_name` to each row by `person_id`. People with no row get `null` for both.
|
|
54
|
+
|
|
55
|
+
`@engine9/schemas/person:transforms:appendName` is the same transform, kept so saved export definitions keep working.
|
|
56
|
+
|
|
57
|
+
## Segment definition clauses
|
|
58
|
+
|
|
59
|
+
`names` matches people by `given_name` and/or `family_name` with `LIKE`. With neither, it selects everyone who has a name row.
|
|
60
|
+
|
|
61
|
+
## Upgrading accounts created before this plugin
|
|
62
|
+
|
|
63
|
+
Before this plugin, the names were `person.given_name` and `person.family_name`. Run the core upgrade once per account after installing this version of `@engine9/schemas`:
|
|
64
|
+
|
|
65
|
+
- `PluginWorker.upgrade()` (server: `e9 personworker upgrade -a <account_id>`)
|
|
66
|
+
- `npx e9core upgrade --db <connection>` for a SQLite or MySQL database
|
|
67
|
+
- `npx e9core upgrade --url https://<site>` for a Cloudflare site (it posts to `/api/admin/upgrade` with `E9_ADMIN_API_KEY`)
|
|
68
|
+
|
|
69
|
+
The upgrade installs this plugin and copies every non-blank name into `person_name`. It then refreshes the `person` plugin's inbound snapshot, so later loads write `person_name` only. The columns stay on `person` with the names as of the upgrade. Dropping them is a separate, opt-in step (`drop_name_columns`), for when nothing reads them. On a PII-free account the upgrade does not install this plugin, and the opt-in drop removes the columns only when they are empty, or when you also pass `discard_names`.
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import schema from './schema.js';
|
|
2
|
+
import upsertPersonName from './transforms/inbound/upsert_tables.js';
|
|
3
|
+
import appendName from './transforms/outbound/appendName.js';
|
|
4
|
+
import { segmentDefinitionClauses } from './segmentDefinitionClauses.js';
|
|
5
|
+
const metadata = {
|
|
6
|
+
name: '@engine9/schemas/person_name',
|
|
7
|
+
unique: true,
|
|
8
|
+
dependencies: {
|
|
9
|
+
'@engine9/schemas/person': '>=1.9.0'
|
|
10
|
+
},
|
|
11
|
+
// Inbound people pipeline slots -> transform export keys (woven by core when installed)
|
|
12
|
+
inbound: {
|
|
13
|
+
upsert: ['upsertPersonName']
|
|
14
|
+
}
|
|
15
|
+
};
|
|
16
|
+
export const transforms = {
|
|
17
|
+
upsertPersonName,
|
|
18
|
+
appendName
|
|
19
|
+
};
|
|
20
|
+
export { metadata };
|
|
21
|
+
export { schema };
|
|
22
|
+
export { segmentDefinitionClauses };
|
|
23
|
+
export default {
|
|
24
|
+
metadata,
|
|
25
|
+
schema,
|
|
26
|
+
segmentDefinitionClauses,
|
|
27
|
+
transforms
|
|
28
|
+
};
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
export const tables = [
|
|
2
|
+
{
|
|
3
|
+
name: 'person_name',
|
|
4
|
+
columns: {
|
|
5
|
+
person_id: 'person_id',
|
|
6
|
+
given_name: 'string',
|
|
7
|
+
family_name: 'string',
|
|
8
|
+
source_input_id: 'foreign_uuid',
|
|
9
|
+
created_at: 'created_at',
|
|
10
|
+
modified_at: 'modified_at'
|
|
11
|
+
},
|
|
12
|
+
indexes: [{ columns: ['person_id'], primary: true }]
|
|
13
|
+
}
|
|
14
|
+
];
|
|
15
|
+
export default {
|
|
16
|
+
tables
|
|
17
|
+
};
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
export const names = {
|
|
2
|
+
title: 'Names',
|
|
3
|
+
form: {
|
|
4
|
+
title: 'Names',
|
|
5
|
+
type: 'object',
|
|
6
|
+
properties: {
|
|
7
|
+
givenName: {
|
|
8
|
+
title: 'First Name',
|
|
9
|
+
type: 'string'
|
|
10
|
+
},
|
|
11
|
+
familyName: {
|
|
12
|
+
title: 'Last Name',
|
|
13
|
+
type: 'string'
|
|
14
|
+
}
|
|
15
|
+
},
|
|
16
|
+
required: []
|
|
17
|
+
},
|
|
18
|
+
optionsToEQL(options) {
|
|
19
|
+
const { givenName = '', familyName = '' } = options;
|
|
20
|
+
const text = [];
|
|
21
|
+
const conditions = [];
|
|
22
|
+
if (givenName.length > 0) {
|
|
23
|
+
text.push(`First name like '${givenName}'`);
|
|
24
|
+
conditions.push({
|
|
25
|
+
type: 'LIKE',
|
|
26
|
+
values: [{ ref: { column: 'given_name' } }, { value: { value: givenName } }]
|
|
27
|
+
});
|
|
28
|
+
}
|
|
29
|
+
if (familyName.length > 0) {
|
|
30
|
+
text.push(`Last name like '${familyName}'`);
|
|
31
|
+
conditions.push({
|
|
32
|
+
type: 'LIKE',
|
|
33
|
+
values: [{ ref: { column: 'family_name' } }, { value: { value: familyName } }]
|
|
34
|
+
});
|
|
35
|
+
}
|
|
36
|
+
return {
|
|
37
|
+
text: text.length ? text.join(' and ') : 'Has a name',
|
|
38
|
+
eql: {
|
|
39
|
+
table: 'person_name',
|
|
40
|
+
conditions
|
|
41
|
+
}
|
|
42
|
+
};
|
|
43
|
+
}
|
|
44
|
+
};
|
|
45
|
+
export const segmentDefinitionClauses = { names };
|
|
46
|
+
export default {
|
|
47
|
+
segmentDefinitionClauses
|
|
48
|
+
};
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
import { mergeIntoQueue } from '@engine9/input-tools';
|
|
2
|
+
|
|
3
|
+
export const type = 'upsert';
|
|
4
|
+
export const description =
|
|
5
|
+
'Upsert given_name / family_name into person_name, one row per person. A name key that is absent keeps the stored value; an explicit null clears it.';
|
|
6
|
+
export const bindings = {
|
|
7
|
+
databaseNames: { path: 'sql.query', options: { table: 'person_name', lookup: ['person_id'] } },
|
|
8
|
+
tablesToUpsert: { path: 'sql.tables.upsert' }
|
|
9
|
+
};
|
|
10
|
+
|
|
11
|
+
const NAME_COLUMNS = ['given_name', 'family_name'];
|
|
12
|
+
|
|
13
|
+
// person_id may arrive as number or string from different pipeline stages
|
|
14
|
+
const personKey = (field, value) => (value == null ? '' : String(value));
|
|
15
|
+
|
|
16
|
+
function mergeName(existing, incoming) {
|
|
17
|
+
return {
|
|
18
|
+
...existing,
|
|
19
|
+
...incoming,
|
|
20
|
+
person_id: existing.person_id,
|
|
21
|
+
source_input_id: existing.source_input_id ?? incoming.source_input_id
|
|
22
|
+
};
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
export async function transform({ batch, databaseNames = [], tablesToUpsert }) {
|
|
26
|
+
if (batch.length === 0) return;
|
|
27
|
+
const queue = [];
|
|
28
|
+
batch.forEach((o) => {
|
|
29
|
+
const present = NAME_COLUMNS.filter((column) => Object.hasOwn(o, column));
|
|
30
|
+
if (present.length === 0) return;
|
|
31
|
+
if (!o.person_id) throw new Error('Cannot update name, no person_id');
|
|
32
|
+
const row = { person_id: o.person_id, source_input_id: o.input_id ?? null };
|
|
33
|
+
for (const column of present) row[column] = o[column];
|
|
34
|
+
mergeIntoQueue(queue, row, {
|
|
35
|
+
keyFields: ['person_id'],
|
|
36
|
+
normalizeField: personKey,
|
|
37
|
+
merge: mergeName,
|
|
38
|
+
label: 'person_name'
|
|
39
|
+
});
|
|
40
|
+
});
|
|
41
|
+
if (queue.length === 0) return;
|
|
42
|
+
// Every queued row carries every column (upsertArray rejects mixed key sets),
|
|
43
|
+
// so a name the batch did not mention is filled from the stored row.
|
|
44
|
+
const stored = new Map(databaseNames.map((row) => [String(row.person_id), row]));
|
|
45
|
+
tablesToUpsert.person_name = tablesToUpsert.person_name || [];
|
|
46
|
+
for (const row of queue) {
|
|
47
|
+
const existing = stored.get(String(row.person_id));
|
|
48
|
+
const full = {
|
|
49
|
+
person_id: row.person_id,
|
|
50
|
+
source_input_id: existing?.source_input_id ?? row.source_input_id
|
|
51
|
+
};
|
|
52
|
+
for (const column of NAME_COLUMNS) {
|
|
53
|
+
full[column] = Object.hasOwn(row, column) ? row[column] : (existing?.[column] ?? null);
|
|
54
|
+
}
|
|
55
|
+
tablesToUpsert.person_name.push(full);
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
export default {
|
|
59
|
+
bindings,
|
|
60
|
+
type,
|
|
61
|
+
description,
|
|
62
|
+
transform
|
|
63
|
+
};
|
|
@@ -3,20 +3,16 @@ export const bindings = {
|
|
|
3
3
|
names: {
|
|
4
4
|
path: 'sql.query',
|
|
5
5
|
options: {
|
|
6
|
-
table: '
|
|
7
|
-
columns: ['
|
|
8
|
-
lookup: [
|
|
9
|
-
|
|
10
|
-
personIdField: 'person_id',
|
|
11
|
-
tableLookupField: 'id'
|
|
12
|
-
}
|
|
13
|
-
]
|
|
6
|
+
table: 'person_name',
|
|
7
|
+
columns: ['person_id', 'given_name', 'family_name'],
|
|
8
|
+
lookup: ['person_id'],
|
|
9
|
+
optional: true
|
|
14
10
|
}
|
|
15
11
|
}
|
|
16
12
|
};
|
|
17
13
|
export const transform = ({ batch, names }) => {
|
|
18
14
|
const nameMap = names.reduce((a, b) => {
|
|
19
|
-
a[b.
|
|
15
|
+
a[b.person_id] = a[b.person_id] || b;
|
|
20
16
|
return a;
|
|
21
17
|
}, {});
|
|
22
18
|
batch.forEach((data) => {
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
import assert from 'node:assert/strict';
|
|
2
|
+
import { describe, it } from 'node:test';
|
|
3
|
+
import upsertPersonName from './transforms/inbound/upsert_tables.js';
|
|
4
|
+
import appendName from './transforms/outbound/appendName.js';
|
|
5
|
+
import plugin from './index.js';
|
|
6
|
+
|
|
7
|
+
describe('person_name upsert_tables', () => {
|
|
8
|
+
it('is declared in the inbound upsert slot', () => {
|
|
9
|
+
assert.deepEqual(plugin.metadata.inbound, { upsert: ['upsertPersonName'] });
|
|
10
|
+
assert.equal(plugin.transforms.upsertPersonName.type, 'upsert');
|
|
11
|
+
});
|
|
12
|
+
|
|
13
|
+
it('queues one person_name row with every column', async () => {
|
|
14
|
+
const tablesToUpsert = {};
|
|
15
|
+
await upsertPersonName.transform({
|
|
16
|
+
batch: [{ person_id: 1, input_id: 'input-a', email: 'ada@example.com', given_name: 'Ada' }],
|
|
17
|
+
databaseNames: [],
|
|
18
|
+
tablesToUpsert
|
|
19
|
+
});
|
|
20
|
+
assert.deepEqual(tablesToUpsert.person_name, [
|
|
21
|
+
{ person_id: 1, source_input_id: 'input-a', given_name: 'Ada', family_name: null }
|
|
22
|
+
]);
|
|
23
|
+
assert.equal('person' in tablesToUpsert, false, 'never writes the person table');
|
|
24
|
+
});
|
|
25
|
+
|
|
26
|
+
it('keeps the stored value for a name key the row did not send', async () => {
|
|
27
|
+
const tablesToUpsert = {};
|
|
28
|
+
await upsertPersonName.transform({
|
|
29
|
+
batch: [{ person_id: 2, input_id: 'input-b', family_name: 'Lovelace' }],
|
|
30
|
+
databaseNames: [{ person_id: '2', given_name: 'Ada', family_name: 'Byron', source_input_id: 'input-a' }],
|
|
31
|
+
tablesToUpsert
|
|
32
|
+
});
|
|
33
|
+
assert.deepEqual(tablesToUpsert.person_name, [
|
|
34
|
+
{ person_id: 2, source_input_id: 'input-a', given_name: 'Ada', family_name: 'Lovelace' }
|
|
35
|
+
]);
|
|
36
|
+
});
|
|
37
|
+
|
|
38
|
+
it('clears a name when the key is present and null', async () => {
|
|
39
|
+
const tablesToUpsert = {};
|
|
40
|
+
await upsertPersonName.transform({
|
|
41
|
+
batch: [{ person_id: 3, family_name: null }],
|
|
42
|
+
databaseNames: [{ person_id: 3, given_name: 'Ada', family_name: 'Lovelace', source_input_id: null }],
|
|
43
|
+
tablesToUpsert
|
|
44
|
+
});
|
|
45
|
+
assert.equal(tablesToUpsert.person_name[0].given_name, 'Ada');
|
|
46
|
+
assert.equal(tablesToUpsert.person_name[0].family_name, null);
|
|
47
|
+
});
|
|
48
|
+
|
|
49
|
+
it('merges rows for the same person in one batch, later keys winning', async () => {
|
|
50
|
+
const tablesToUpsert = {};
|
|
51
|
+
await upsertPersonName.transform({
|
|
52
|
+
batch: [
|
|
53
|
+
{ person_id: 4, input_id: 'first', given_name: 'Ada', family_name: 'Byron' },
|
|
54
|
+
{ person_id: '4', input_id: 'second', family_name: 'Lovelace' }
|
|
55
|
+
],
|
|
56
|
+
databaseNames: [],
|
|
57
|
+
tablesToUpsert
|
|
58
|
+
});
|
|
59
|
+
assert.deepEqual(tablesToUpsert.person_name, [
|
|
60
|
+
{ person_id: 4, source_input_id: 'first', given_name: 'Ada', family_name: 'Lovelace' }
|
|
61
|
+
]);
|
|
62
|
+
});
|
|
63
|
+
|
|
64
|
+
it('queues nothing when no row has a name key', async () => {
|
|
65
|
+
const tablesToUpsert = {};
|
|
66
|
+
await upsertPersonName.transform({
|
|
67
|
+
batch: [{ person_id: 5, email: 'ada@example.com' }, { email: 'no-id@example.com' }],
|
|
68
|
+
databaseNames: [],
|
|
69
|
+
tablesToUpsert
|
|
70
|
+
});
|
|
71
|
+
assert.deepEqual(tablesToUpsert, {});
|
|
72
|
+
});
|
|
73
|
+
|
|
74
|
+
it('throws when a name is present and person_id is missing', async () => {
|
|
75
|
+
await assert.rejects(
|
|
76
|
+
() => upsertPersonName.transform({ batch: [{ given_name: 'Ada' }], databaseNames: [], tablesToUpsert: {} }),
|
|
77
|
+
/no person_id/
|
|
78
|
+
);
|
|
79
|
+
});
|
|
80
|
+
});
|
|
81
|
+
|
|
82
|
+
describe('person_name appendName', () => {
|
|
83
|
+
it('appends names by person_id', () => {
|
|
84
|
+
const batch = [{ person_id: 1 }, { person_id: 2 }];
|
|
85
|
+
appendName.transform({ batch, names: [{ person_id: 1, given_name: 'Ada', family_name: 'Lovelace' }] });
|
|
86
|
+
assert.deepEqual(batch, [
|
|
87
|
+
{ person_id: 1, given_name: 'Ada', family_name: 'Lovelace' },
|
|
88
|
+
{ person_id: 2, given_name: null, family_name: null }
|
|
89
|
+
]);
|
|
90
|
+
});
|
|
91
|
+
});
|
package/person_phone/README.md
CHANGED
|
@@ -49,9 +49,9 @@ The inbound `upsert` transform writes rows to `person_phone`.
|
|
|
49
49
|
|
|
50
50
|
`appendPhoneHash` hashes the row's normalized `phone` into `phone_hash_v1` when the hash is not already present.
|
|
51
51
|
|
|
52
|
-
##
|
|
52
|
+
## Segment definition clauses
|
|
53
53
|
|
|
54
|
-
The `phones`
|
|
54
|
+
The `phones` clause selects people who have a phone number, optionally filtered by:
|
|
55
55
|
|
|
56
56
|
- `smsStatus`, using the same values as `sms_status`
|
|
57
57
|
- `callStatus`, using the same values as `call_status`
|
package/person_phone/index.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import schema from './schema.js';
|
|
2
|
-
import
|
|
2
|
+
import { segmentDefinitionClauses } from './segmentDefinitionClauses.js';
|
|
3
3
|
import segments from './segments.js';
|
|
4
4
|
import extractPhoneHashes from './transforms/inbound/extract_identifiers.js';
|
|
5
5
|
import upsertPersonPhone from './transforms/inbound/upsert_tables.js';
|
|
@@ -22,12 +22,12 @@ export const transforms = {
|
|
|
22
22
|
};
|
|
23
23
|
export { metadata };
|
|
24
24
|
export { schema };
|
|
25
|
-
export {
|
|
25
|
+
export { segmentDefinitionClauses };
|
|
26
26
|
export { segments };
|
|
27
27
|
export default {
|
|
28
28
|
metadata,
|
|
29
29
|
schema,
|
|
30
30
|
transforms,
|
|
31
|
-
|
|
31
|
+
segmentDefinitionClauses,
|
|
32
32
|
segments
|
|
33
33
|
};
|
package/person_phone/segments.js
CHANGED
package/person_remote/index.js
CHANGED
|
@@ -14,8 +14,7 @@ const metadata = {
|
|
|
14
14
|
upsert: ['upsertPersonRemote']
|
|
15
15
|
}
|
|
16
16
|
};
|
|
17
|
-
|
|
18
|
-
all: {
|
|
17
|
+
const all = {
|
|
19
18
|
title: 'Remote people',
|
|
20
19
|
name: 'All remote people',
|
|
21
20
|
form: {
|
|
@@ -39,8 +38,8 @@ export const search = {
|
|
|
39
38
|
}
|
|
40
39
|
]
|
|
41
40
|
})
|
|
42
|
-
}
|
|
43
|
-
};
|
|
41
|
+
};
|
|
42
|
+
export const segmentDefinitionClauses = { all };
|
|
44
43
|
export const transforms = {
|
|
45
44
|
extractRemotePersonIds,
|
|
46
45
|
upsertPersonRemote,
|
|
@@ -53,6 +52,6 @@ export default {
|
|
|
53
52
|
metadata,
|
|
54
53
|
schema,
|
|
55
54
|
metrics,
|
|
56
|
-
|
|
55
|
+
segmentDefinitionClauses,
|
|
57
56
|
transforms
|
|
58
57
|
};
|
package/segment/index.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import schema from './schema.js';
|
|
2
|
-
import
|
|
2
|
+
import { segmentDefinitionClauses } from './segmentDefinitionClauses.js';
|
|
3
3
|
import upsert from './transforms/inbound/upsert.js';
|
|
4
4
|
const metadata = {
|
|
5
5
|
name: '@engine9/schemas/segment',
|
|
@@ -12,10 +12,10 @@ export const transforms = {
|
|
|
12
12
|
};
|
|
13
13
|
export { metadata };
|
|
14
14
|
export { schema };
|
|
15
|
-
export {
|
|
15
|
+
export { segmentDefinitionClauses };
|
|
16
16
|
export default {
|
|
17
17
|
metadata,
|
|
18
18
|
schema,
|
|
19
|
-
|
|
19
|
+
segmentDefinitionClauses,
|
|
20
20
|
transforms
|
|
21
21
|
};
|
package/segment/schema.js
CHANGED
|
@@ -23,15 +23,15 @@ export const tables = [
|
|
|
23
23
|
category: 'string', // A arbitrary type, user facing, not to be used for logic
|
|
24
24
|
definition_path: 'string', // a definition file path, will be used for configuring
|
|
25
25
|
stars: 'int', //number of stars, good for UI organizing
|
|
26
|
-
|
|
26
|
+
definition: 'json',
|
|
27
27
|
build_type: {
|
|
28
28
|
type: 'enum',
|
|
29
29
|
nullable: false,
|
|
30
30
|
default_value: 'list',
|
|
31
31
|
values: [
|
|
32
32
|
'list', //static list of people
|
|
33
|
-
'
|
|
34
|
-
'
|
|
33
|
+
'definition', // built by the segment definition
|
|
34
|
+
'scheduled_definition', // built on a schedule by the segment definition
|
|
35
35
|
'remote_count', // no person records, just counts, pulled from remote site
|
|
36
36
|
'remote', // pulled from remote plugin site
|
|
37
37
|
'manual' // created outside normal processing
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@engine9/schemas/stacks/limited-pii",
|
|
3
|
-
"description": "PII-minimized Engine9 stack: hash identifiers instead of plaintext email, phone, or postal address. Excludes person_email, person_phone, and
|
|
4
|
-
"version": "1.
|
|
3
|
+
"description": "PII-minimized Engine9 stack: hash identifiers instead of plaintext names, email, phone, or postal address. Excludes person_name, person_email, person_phone, person_address, and transaction/contact_details even if requested.",
|
|
4
|
+
"version": "1.2.0",
|
|
5
5
|
"unique": true,
|
|
6
6
|
"include": [
|
|
7
7
|
"@engine9/schemas/plugin",
|
|
@@ -11,12 +11,13 @@
|
|
|
11
11
|
"@engine9/schemas/timeline",
|
|
12
12
|
"@engine9/schemas/source_code",
|
|
13
13
|
"@engine9/schemas/transaction/core",
|
|
14
|
-
"@engine9/schemas/transaction/contact_details",
|
|
15
14
|
"@engine9/schemas/person_hash"
|
|
16
15
|
],
|
|
17
16
|
"exclude": [
|
|
17
|
+
"@engine9/schemas/person_name",
|
|
18
18
|
"@engine9/schemas/person_email",
|
|
19
19
|
"@engine9/schemas/person_phone",
|
|
20
|
-
"@engine9/schemas/person_address"
|
|
20
|
+
"@engine9/schemas/person_address",
|
|
21
|
+
"@engine9/schemas/transaction/contact_details"
|
|
21
22
|
]
|
|
22
23
|
}
|
package/stacks/stacks.test.js
CHANGED
|
@@ -5,6 +5,8 @@ import { metadata as limitedPii } from './limited-pii/index.js';
|
|
|
5
5
|
|
|
6
6
|
test('standard stack includes plaintext contact plugins and does not exclude them', () => {
|
|
7
7
|
assert.equal(standard.name, '@engine9/schemas/stacks/standard');
|
|
8
|
+
assert.ok(standard.include.includes('@engine9/schemas/person_name'));
|
|
9
|
+
assert.ok(standard.include.includes('@engine9/schemas/transaction/contact_details'));
|
|
8
10
|
assert.ok(standard.include.includes('@engine9/schemas/person_email'));
|
|
9
11
|
assert.ok(standard.include.includes('@engine9/schemas/person_phone'));
|
|
10
12
|
assert.ok(standard.include.includes('@engine9/schemas/person_address'));
|
|
@@ -18,6 +20,10 @@ test('standard stack includes plaintext contact plugins and does not exclude the
|
|
|
18
20
|
test('limited-pii stack includes person_hash and excludes plaintext contact plugins', () => {
|
|
19
21
|
assert.equal(limitedPii.name, '@engine9/schemas/stacks/limited-pii');
|
|
20
22
|
assert.ok(limitedPii.include.includes('@engine9/schemas/person_hash'));
|
|
23
|
+
for (const path of ['@engine9/schemas/person_name', '@engine9/schemas/transaction/contact_details']) {
|
|
24
|
+
assert.ok(!limitedPii.include.includes(path), `${path} is not included`);
|
|
25
|
+
assert.ok(limitedPii.exclude.includes(path), `${path} is excluded`);
|
|
26
|
+
}
|
|
21
27
|
assert.ok(!limitedPii.include.includes('@engine9/schemas/person_email'));
|
|
22
28
|
assert.ok(!limitedPii.include.includes('@engine9/schemas/person_phone'));
|
|
23
29
|
assert.ok(!limitedPii.include.includes('@engine9/schemas/person_address'));
|
|
@@ -1,11 +1,12 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@engine9/schemas/stacks/standard",
|
|
3
|
-
"description": "Opt-in Engine9 schema stack: person, plaintext email, phone, and postal address, timeline, source codes, and transactions. Reports install separately.",
|
|
4
|
-
"version": "1.
|
|
3
|
+
"description": "Opt-in Engine9 schema stack: person, plaintext names, email, phone, and postal address, timeline, source codes, and transactions. Reports install separately.",
|
|
4
|
+
"version": "1.1.0",
|
|
5
5
|
"unique": true,
|
|
6
6
|
"include": [
|
|
7
7
|
"@engine9/schemas/plugin",
|
|
8
8
|
"@engine9/schemas/person",
|
|
9
|
+
"@engine9/schemas/person_name",
|
|
9
10
|
"@engine9/schemas/person_remote",
|
|
10
11
|
"@engine9/schemas/segment",
|
|
11
12
|
"@engine9/schemas/person_email",
|
|
@@ -1,9 +1,11 @@
|
|
|
1
|
-
// Contact details captured on a transaction: address, email, phone, employer, and occupation.
|
|
1
|
+
// Contact details captured on a transaction: payer name, address, email, phone, employer, and occupation.
|
|
2
2
|
export const tables = [
|
|
3
3
|
{
|
|
4
4
|
name: 'transaction',
|
|
5
5
|
prefix: false,
|
|
6
6
|
columns: {
|
|
7
|
+
given_name: 'string',
|
|
8
|
+
family_name: 'string',
|
|
7
9
|
street_1: 'string',
|
|
8
10
|
street_2: 'string',
|
|
9
11
|
city: 'string',
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# Transaction Core Schema Plugin
|
|
2
2
|
|
|
3
|
-
The `@engine9/schemas/transaction/core` plugin stores payment and gift transactions for people and provides the common transaction
|
|
3
|
+
The `@engine9/schemas/transaction/core` plugin stores payment and gift transactions for people and provides the common transaction segment definition clauses, metrics, and segment used by imports, reports, and exports.
|
|
4
4
|
|
|
5
5
|
It depends on `@engine9/schemas/person` because every transaction belongs to a person through `person_id`.
|
|
6
6
|
|
|
@@ -8,6 +8,8 @@ It depends on `@engine9/schemas/person` because every transaction belongs to a p
|
|
|
8
8
|
|
|
9
9
|
The plugin creates the `transaction` table. Each row is a dated amount tied to a person, an input, and optional source-code / message attribution.
|
|
10
10
|
|
|
11
|
+
The payer's name, email, phone, and address are not part of this plugin. `@engine9/schemas/transaction/contact_details` adds those columns to `transaction`; the limited-pii stack excludes it.
|
|
12
|
+
|
|
11
13
|
| Field | Type | Description |
|
|
12
14
|
| --- | --- | --- |
|
|
13
15
|
| `id` | `id_uuid` | Primary row identifier. |
|
|
@@ -22,7 +24,6 @@ The plugin creates the `transaction` table. Each row is a dated amount tied to a
|
|
|
22
24
|
| `recurs_id` | `int` | Recurring frequency. `0` is one-time. Defaults to `0`. |
|
|
23
25
|
| `recurring_number` | `int` | Sequence number in a recurring series. |
|
|
24
26
|
| `refund_amount` | `currency` | Refunded amount when applicable. |
|
|
25
|
-
| `given_name` / `family_name` / `email` | `string` | Snapshot of payer identity on the transaction. |
|
|
26
27
|
| `source_code_id` | `source_code_id` | Source code captured on the transaction. |
|
|
27
28
|
| `override_source_code_id` | `source_code_id` | Manual source-code override. |
|
|
28
29
|
| `final_source_code_id` | `source_code_id` | Source code after attribution. |
|
|
@@ -52,10 +53,10 @@ The inbound `upsert` transform writes rows to `transaction`.
|
|
|
52
53
|
|
|
53
54
|
`appendTransactionSummary` looks up a person's transactions and attaches summary fields such as count, revenue, first/last/smallest/largest gifts, and recurring totals.
|
|
54
55
|
|
|
55
|
-
##
|
|
56
|
+
## Segment definition clauses
|
|
56
57
|
|
|
57
|
-
- `all`
|
|
58
|
-
- `minimum`
|
|
58
|
+
- `all` selects people who have any transaction, optionally filtered by `plugin_id` and a `start` / `end` window on `ts` (ISO datetimes or relative expressions such as `-30d`).
|
|
59
|
+
- `minimum` selects people who have a transaction of at least a given `amount`.
|
|
59
60
|
|
|
60
61
|
## Metrics
|
|
61
62
|
|
|
@@ -73,7 +74,7 @@ Predefined audiences shipped with this schema plugin. On install, each key becom
|
|
|
73
74
|
| **Definition path** | `@engine9/schemas/transaction/core:segments:customers` |
|
|
74
75
|
| **Who is included** | People who have at least one row in `transaction` |
|
|
75
76
|
| **Who is excluded** | People with no transactions |
|
|
76
|
-
| **How it is built** |
|
|
77
|
+
| **How it is built** | The `all` clause, which selects `person_id` from `transaction` with no extra filters |
|
|
77
78
|
| **Universe** | None — any transaction from any plugin, date, or amount counts |
|
|
78
79
|
|
|
79
80
|
Refunds, recurring gifts, and $0 rows are not filtered out. If a person has ever appeared on a transaction row, they are a customer.
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import schema from './schema.js';
|
|
2
|
-
import
|
|
2
|
+
import { segmentDefinitionClauses } from './segmentDefinitionClauses.js';
|
|
3
3
|
import metrics from './metrics.js';
|
|
4
4
|
import segments from './segments.js';
|
|
5
5
|
import upsert from './transforms/inbound/upsert_tables.js';
|
|
@@ -19,13 +19,13 @@ export const transforms = {
|
|
|
19
19
|
export { metadata };
|
|
20
20
|
export { schema };
|
|
21
21
|
export { metrics };
|
|
22
|
-
export {
|
|
22
|
+
export { segmentDefinitionClauses };
|
|
23
23
|
export { segments };
|
|
24
24
|
export default {
|
|
25
25
|
metadata,
|
|
26
26
|
schema,
|
|
27
27
|
metrics,
|
|
28
28
|
transforms,
|
|
29
|
-
|
|
29
|
+
segmentDefinitionClauses,
|
|
30
30
|
segments
|
|
31
31
|
};
|
|
@@ -18,9 +18,6 @@ export const tables = [
|
|
|
18
18
|
},
|
|
19
19
|
recurring_number: 'int',
|
|
20
20
|
refund_amount: 'currency',
|
|
21
|
-
given_name: 'string',
|
|
22
|
-
family_name: 'string',
|
|
23
|
-
email: 'string',
|
|
24
21
|
source_code_id: 'source_code_id',
|
|
25
22
|
override_source_code_id: 'source_code_id',
|
|
26
23
|
final_source_code_id: 'source_code_id',
|
|
@@ -4,7 +4,7 @@ export const settings = [
|
|
|
4
4
|
type: 'boolean',
|
|
5
5
|
default: false,
|
|
6
6
|
description:
|
|
7
|
-
'When true, installDefaultPlugins prefers @engine9/schemas/stacks/limited-pii over default_stack, and installs of stacks/standard, person_email, person_phone, and
|
|
7
|
+
'When true, installDefaultPlugins prefers @engine9/schemas/stacks/limited-pii over default_stack, and installs of stacks/standard, person_name, person_email, person_phone, person_address, and transaction/contact_details are refused. Does not drop existing plaintext tables. Also omits the plaintext person tables from the default export list when tables is not set.'
|
|
8
8
|
}
|
|
9
9
|
];
|
|
10
10
|
export default {
|