@atmem/architect 0.0.0-stage → 0.2.0

package/src/invoice-ibm-operations.mjs

@@ -0,0 +1,278 @@
+import{v,r,l,UUID}from'./invoice-builder.mjs';
+
+// IBM 2.9 supports JobDefinition, not a Terraform cron subscription resource.
+// Keep job delivery separate from scheduling/alert acceptance under qualification.
+export function ibmInvoiceOperations(b,n,{projectId,pull}){
+ const{add}=b;
+ b.variable.operations_enabled={type:'bool',default:false,description:'Create the three fixed SQL operating tasks after reviewed immutable image publication. Scheduling and alert acceptance are separate.'};
+ b.examples.operations_enabled=false;b.testExpressions.operations_enabled='true';
+ b.variable.maintenance_enabled={type:'bool',default:false,description:'Enable operating alerts after bootstrap; separately install the owned cron sources and verify successful runs before application release.'};b.examples.maintenance_enabled=false;b.testExpressions.maintenance_enabled='true';
+ const logs=b.input('operator_logs_instance_id','Existing reviewed same-region Cloud Logs instance receiving this region\'s Code Engine platform logs. Existing account routing is preserved.','88888888-8888-4888-8888-888888888888',UUID);
+ const webhook=b.input('operator_notification_integration_id','Existing verified Cloud Logs outgoing integration numeric ID; verify intended on-call recipients/delivery without exposing its endpoint or credentials.','12345','^[1-9][0-9]{0,9}$');
+ b.variable.operator_notification_integration_id.validation.condition='${can(regex("^[1-9][0-9]{0,9}$",var.operator_notification_integration_id)) && try(tonumber(var.operator_notification_integration_id),0) <= 4294967295}';
+ b.resource.terraform_data.review_gate.lifecycle.precondition.push({condition:'${!var.maintenance_enabled || var.operations_enabled}',error_message:'Operating alerts and schedules require fixed operator tasks.'},{condition:'${!var.applications_enabled || (var.operations_enabled && var.maintenance_enabled)}',error_message:'Initialize SQL and verify operating schedules, log routing and notifications before application release.'});
+ b.locals.operator_deployment=b.output.operator_deployment.value;
+ b.locals.operator_secret_name=l('name')+'-operator-sql';
+ const literal=(name,value)=>({name,value,type:'literal'});
+ const reference=(name,type,key,reference)=>({name,type,key,reference});
+ for(const action of ['initialize','maintain','monitor']){
+  const env=[literal('DATABASE_NAME','ibmclouddb'),literal('OPERATOR_DEPLOYMENT_BASE64','${base64encode(jsonencode(local.operator_deployment))}'),...Object.entries({DATABASE_HOST:'host',DATABASE_PORT:'port',DATABASE_CA_PEM:'ca_pem',DATABASE_USER:'operator_user'}).map(([name,key])=>reference(name,'config_map_key_reference',key,l('database_config_name'))),reference('DATABASE_PASSWORD','secret_key_reference','password',l('operator_secret_name'))];
+  // Bootstrap needs the two runtime passwords, never the worker's accounting token.
+  if(action==='initialize')for(const purpose of ['api','worker'])env.push(reference(purpose.toUpperCase()+'_DATABASE_PASSWORD','secret_key_reference','password',l(purpose==='api'?'api_secret_name':'worker_secret_name')));
+  add('ibm_code_engine_job',n+'_operator_'+action,{count:'${var.operations_enabled ? 1 : 0}',project_id:projectId,name:l('name')+'-'+action,image_reference:v('runtime_image'),image_secret:pull,run_commands:['python','-I','/app/invoice_operator_job.py'],run_arguments:[action,'--execute'],run_mode:'task',run_as_user:10001,run_service_account:'none',run_compute_resource_token_enabled:false,scale_array_spec:'0',scale_cpu_limit:'1',scale_memory_limit:'1G',scale_max_execution_time:960,scale_retry_limit:0,run_env_variables:env,depends_on:['ibm_database.'+n]});
+ }
+ const owned='app:"codeengine" AND codeengine.projectGuid:"'+projectId+'" AND codeengine.region:"'+b.region+'"';
+ const query=(action,status)=>owned+' AND codeengine.component:"'+l('name')+'-'+action+'" AND message:"invoice-operator-v1" AND message:"'+status+'"';
+ const failures=owned+' AND ('+['initialize','maintain','monitor'].map(action=>'codeengine.component:"'+l('name')+'-'+action+'"').join(' OR ')+') AND ((message:"invoice-operator-v1" AND (message:"failed" OR message:"attention" OR message:"backlog")) OR level:"error" OR level:"fatal")';
+ const definitions=[['failed',failures,'more_than_or_unspecified',0,'minutes_5_or_unspecified'],['monitor_missing',query('monitor','ok'),'less_than',1,'hours_2'],['maintenance_missing',query('maintain','ok'),'less_than',1,'hours_36']];
+ b.output.operator_log_queries={value:Object.fromEntries(definitions.map(([purpose,filter])=>[purpose,filter]))};
+ for(const[purpose,filter,condition,threshold,window]of definitions){
+  add('ibm_logs_alert_definition',n+'_'+purpose,{instance_id:logs,region:b.region,name:l('name')+'-'+purpose,description:'Owned SQL operator '+purpose+'. Verify native log samples, alert evaluation and existing on-call delivery before release; no customer invoice data in notifications.',enabled:v('maintenance_enabled'),phantom_mode:false,priority:'p2',type:'logs_threshold',logs_threshold:{condition_type:condition,evaluation_delay_ms:60000,notification_payload_filter:[],logs_filter:{simple_filter:{lucene_query:filter}},rules:[{condition:{threshold,time_window:{logs_time_window_specific_value:window}},override:{priority:'p2'}}],undetected_values_management:{auto_retire_timeframe:'never_or_unspecified',trigger_undetected_values:condition==='less_than'}},notification_group:{webhooks:[{minutes:1,notify_on:'triggered_and_resolved',integration:{integration_id:'${tonumber(var.operator_notification_integration_id)}'}}]},incidents_settings:{minutes:10,notify_on:'triggered_and_resolved'}});
+  b.files['monitoring/'+purpose+'.lucene']=filter+'\n';
+ }
+ b.output.operator_jobs={value:{account:v('account_id'),region:b.region,projectId,maintenanceEnabled:v('maintenance_enabled'),operatorSecret:l('operator_secret_name'),names:Object.fromEntries(['initialize','maintain','monitor'].map(action=>[action,l('name')+'-'+action]))}};
+ b.files['scripts/run-operator.py']=IBM_OPERATOR_RUNNER;
+ b.files['OPERATING_JOBS.md']=IBM_OPERATING_JOBS;
+ b.assertions.push(`run "operator_before_applications" {
+ command = plan
+ variables {
+ applications_enabled = false
+ maintenance_enabled = false
+ }
+ assert {
+ condition = length(ibm_code_engine_job.${n}_operator_initialize) == 1 && length(ibm_code_engine_app.${n}_api) == 0 && ibm_code_engine_job.${n}_operator_monitor[0].run_service_account == "none" && !ibm_code_engine_job.${n}_operator_monitor[0].run_compute_resource_token_enabled && ibm_code_engine_job.${n}_operator_monitor[0].scale_retry_limit == 0
+ error_message = "Bootstrap is separate, bounded and without application/cloud authority."
+ }
+}
+run "operator_mutable_image_rejected" {
+ command = plan
+ variables {
+ applications_enabled = false
+ runtime_image = "unreviewed:latest"
+ maintenance_enabled = false
+ }
+ expect_failures = [var.runtime_image]
+}
+run "operating_jobs_required" {
+ command = plan
+ variables {
+ operations_enabled = false
+ maintenance_enabled = false
+ }
+ expect_failures = [terraform_data.review_gate]
+}
+run "maintenance_without_jobs_rejected" {
+ command = plan
+ variables {
+ applications_enabled = false
+ operations_enabled = false
+ }
+ expect_failures = [terraform_data.review_gate]
+}
+run "notification_id_overflow_rejected" {
+ command = plan
+ variables { operator_notification_integration_id = "4294967296" }
+ expect_failures = [var.operator_notification_integration_id]
+}
+`);
+}
+
+const IBM_OPERATING_JOBS=`# IBM fixed invoice operating tasks
+
+This package delivers initialize/maintain/monitor task definitions, an explicit
+owned cron installer/observer, and Cloud Logs alert definitions. Local tests do not
+prove real native execution, log routing, notification delivery or deployment.
+
+Foundation uses applications_enabled=false, operations_enabled=false and
+maintenance_enabled=false. Publish
+the reviewed immutable runtime image into the owned registry only when authorized.
+Set operations_enabled=true with applications still false to create fixed tasks.
+Populate the separate operator SQL secret outside Terraform. databaseConfig also
+needs operator_user: the exact administrative login for this new owned database.
+Only initialize receives the separate API and worker SQL password keys. No task
+receives the accounting token, parser token, model or object-store authority.
+
+The destination operator installs/authenticates IBM Cloud CLI and its reviewed Code
+Engine plugin in a dedicated private CLI home. For example, set IBMCLOUD_HOME to
+this package's .local/ibm-operator-cli during login/plugin setup. Keep that directory
+mode 0700 and outside source control. Never copy a user's global CLI credentials.
+
+Preview: python3 scripts/run-operator.py initialize
+Authorized execution adds --execute. The helper reads this package's Terraform
+output, explicitly selects its account/region/project in the isolated CLI home,
+then submits the fixed task with --wait. It supplies no image, environment, command
+or service-account overrides. Job-launch permission is privileged; do not grant it
+to the application, worker, model or public user. Retain the actual completed job
+and initialized:true/monitor/maintenance receipt, not merely submit acceptance.
+
+After bootstrap, run maintain and monitor once and inspect actual successful
+receipts in the existing Cloud Logs destination. Verify that regional Code Engine
+platform logging is routed to operator_logs_instance_id through the existing
+account routing. Review its region/data custody and the existing outgoing
+notification integration. Do not replace account log routing or expose credentials.
+Read terraform output -json operator_log_queries for resolved queries; monitoring/
+files are readable Terraform expression templates, not resolved instance selectors.
+The Lucene filters use documented app/codeengine.projectGuid/region/component and
+message/level fields, never deprecated label.Namespace/Project/Stream fields.
+Verify the actual native samples match each emitted query before enabling alerts;
+account parsing/routing can change message layout. Missing SQL monitor alerts after
+two hours; missing daily maintenance after 36 hours. Undetected values trigger
+missing-run alerts; native zero-sample evaluation/delivery still need acceptance.
+Notifications exclude payload fields; no invoice/subject/amount is emitted by jobs.
+
+Keep applications false, set maintenance_enabled=true, apply reviewed alerts, then
+preview python3 scripts/run-operator.py install-schedules. With authorized --execute,
+the helper uses supported Knative PingSource and IBM JobDefinition APIs in this
+project's isolated KUBECONFIG. Only maintain (02:17 UTC daily) and monitor (hourly at
+minute 7) are scheduled. Initialize is never scheduled. It refuses unrelated source
+ownership and uses native resourceVersion to avoid overwriting concurrent changes.
+Use check-schedules --execute to observe configuration and current Ready generation.
+Ready means reconciled source configuration, not successful SQL work. Retain actual
+successful scheduled task/log/alert delivery evidence before application release.
+Cron sources are explicitly operated resources outside Terraform because IBM 2.9
+has no cron subscription resource; do not hide account writes in a provisioner.
+Before rollback, identify the two exact owned PingSource names/UIDs, verify their
+owner references and remove only those sources with the authorized native operator;
+do not delete the namespace/project. Disable alerts during an approved maintenance
+window. Installed sources persist independently of the Terraform boolean: changing
+maintenance_enabled=false does not pause or delete an installed source.
+
+Each task uses one array index, no automatic retries and a 960-second platform
+limit; the application enforces its own 900-second work deadline. Private SQL TLS,
+actual native login privileges, task outcome, schedules and alert delivery still
+need separately authorized destination checks. Provider schema/mocks are not
+deployment, SQL runtime or scheduling evidence.
+`;
+
+const IBM_OPERATOR_RUNNER=`"""Preview or explicitly execute a fixed owned IBM Code Engine task."""
+import argparse
+from contextlib import contextmanager
+import copy
+import fcntl
+import json
+import os
+from pathlib import Path
+import re
+import stat
+import subprocess
+
+ROOT=Path(__file__).resolve().parents[1]
+
+def unique_object(pairs):
+    value={}
+    for key,child in pairs:
+        if key in value:raise ValueError('Duplicate operator output field')
+        value[key]=child
+    return value
+
+def configuration():
+    result=subprocess.run(['terraform','output','-json','operator_jobs'],cwd=ROOT,check=True,capture_output=True,text=True,timeout=30)
+    value=json.loads(result.stdout,object_pairs_hook=unique_object)
+    if not isinstance(value,dict)or set(value)!={'account','region','projectId','maintenanceEnabled','operatorSecret','names'}or type(value['maintenanceEnabled'])is not bool:raise ValueError('Use the owned operator_jobs output')
+    if not re.fullmatch(r'[0-9a-f]{32}',value['account'])or not re.fullmatch(r'[a-z]{2}-[a-z]+',value['region'])or not re.fullmatch(r'[0-9a-f]{8}(-[0-9a-f]{4}){3}-[0-9a-f]{12}',value['projectId']):raise ValueError('Invalid native destination')
+    names=value['names']
+    if not isinstance(names,dict)or set(names)!={'initialize','maintain','monitor'}:raise ValueError('Use all three fixed jobs')
+    prefix=None
+    for action,name in names.items():
+        if not isinstance(name,str)or not re.fullmatch(r'invoice-[0-9a-f]{12}-'+action,name):raise ValueError('Use a fixed owned task')
+        current=name.removesuffix('-'+action)
+        if prefix is not None and current!=prefix:raise ValueError('Mixed deployment jobs')
+        prefix=current
+    return value
+
+def commands(value,action):
+    return [['ibmcloud','target','-c',value['account'],'-r',value['region']],['ibmcloud','ce','project','select','--id',value['projectId'],'--quiet'],['ibmcloud','ce','jobrun','submit','--job',value['names'][action],'--wait','--wait-timeout','1000']]
+
+@contextmanager
+def private_context():
+    home=ROOT/'.local/ibm-operator-cli'
+    # Pre-existing private CLI authentication is required; do not silently adopt
+    # global account context or follow a symlink to somebody else's CLI state.
+    metadata=home.lstat()
+    if not stat.S_ISDIR(metadata.st_mode)or metadata.st_uid!=os.geteuid()or metadata.st_mode&0o077:raise ValueError('Authenticate in the owned mode-0700 CLI directory first')
+    lock=os.open(home/'operator.lock',os.O_CREAT|os.O_RDWR|os.O_NOFOLLOW,0o600)
+    with os.fdopen(lock,'w')as stream:
+        metadata=os.fstat(stream.fileno())
+        if not stat.S_ISREG(metadata.st_mode)or metadata.st_uid!=os.geteuid()or metadata.st_mode&0o077:raise ValueError('Use a private owned operation lock')
+        fcntl.flock(stream,fcntl.LOCK_EX|fcntl.LOCK_NB)
+        kube=home/'operator-kubeconfig'
+        if kube.exists()or kube.is_symlink():
+            metadata=kube.lstat()
+            if not stat.S_ISREG(metadata.st_mode)or metadata.st_uid!=os.geteuid()or metadata.st_mode&0o077:raise ValueError('Use a private owned kubeconfig')
+        env={**os.environ,'IBMCLOUD_HOME':str(home),'IBMCLOUD_TRACE':'false','KUBECONFIG':str(kube)}
+        previous=os.umask(0o077)
+        try:yield env
+        finally:os.umask(previous)
+
+def kubectl(env,arguments,body=None):
+    result=subprocess.run(['kubectl',*arguments],cwd=ROOT,env=env,check=True,capture_output=True,text=True,input=json.dumps(body)if body is not None else None,timeout=150)
+    if len(result.stdout)>1000000:raise ValueError('Unexpected native resource response')
+    return result.stdout.strip()
+
+def get_resource(env,kind,name,namespace,optional=False):
+    raw=kubectl(env,['get',kind,name,'--namespace',namespace,'-o','json',*(['--ignore-not-found']if optional else[])])
+    if optional and not raw:return None
+    return json.loads(raw,object_pairs_hook=unique_object)
+
+def ownership(value,job,namespace,prefix,name):
+    metadata=value.get('metadata',{});references=metadata.get('ownerReferences',[])
+    expected={'apiVersion':'codeengine.cloud.ibm.com/v1beta1','kind':'JobDefinition','name':job['metadata']['name'],'uid':job['metadata']['uid'],'controller':False,'blockOwnerDeletion':False}
+    if value.get('apiVersion')!='sources.knative.dev/v1'or value.get('kind')!='PingSource'or metadata.get('namespace')!=namespace or metadata.get('name')!=name or metadata.get('labels',{}).get('atmem.ai/operator-package')!=prefix or expected not in references:raise ValueError('Refuse unrelated cron source ownership')
+    if len(references)!=1 or not metadata.get('uid')or not metadata.get('resourceVersion'):raise ValueError('Ambiguous cron source identity')
+
+def schedule_spec(job,action):
+    return {'schedule':'17 2 * * *'if action=='maintain'else'7 * * * *','timezone':'UTC','contentType':'application/json','data':'{}','sink':{'ref':{'apiVersion':'codeengine.cloud.ibm.com/v1beta1','kind':'JobDefinition','name':job['metadata']['name']}}}
+
+def verify_ready(value,expected):
+    spec=value.get('spec',{})
+    if any(spec.get(key)!=child for key,child in expected.items()):raise ValueError('Cron configuration differs from the owned task contract')
+    status=value.get('status',{});generation=value['metadata'].get('generation')
+    if type(generation)is not int or status.get('observedGeneration')!=generation or not any(c.get('type')=='Ready'and c.get('status')=='True'for c in status.get('conditions',[])):raise ValueError('Cron source has not reconciled this generation')
+
+def schedules(value,env,install):
+    if install and not value['maintenanceEnabled']:raise ValueError('Enable reviewed operating alerts before installing schedules')
+    target=commands(value,'monitor')[:2];target[1]=[*target[1],'--kubecfg']
+    for command in target:subprocess.run(command,cwd=ROOT,env=env,check=True,timeout=120)
+    namespace=kubectl(env,['config','view','--minify','-o','jsonpath={.contexts[0].context.namespace}'])
+    if not re.fullmatch(r'[a-z0-9][a-z0-9-]{0,62}',namespace):raise ValueError('Use the selected native project namespace')
+    prefix=value['names']['monitor'].removesuffix('-monitor');result=[]
+    # Observe both targets and ownership boundaries before making either write.
+    pending=[]
+    for action in ['maintain','monitor']:
+        job=get_resource(env,'jobdefinitions.codeengine.cloud.ibm.com',value['names'][action],namespace)
+        metadata=job.get('metadata',{})
+        if job.get('apiVersion')!='codeengine.cloud.ibm.com/v1beta1'or job.get('kind')!='JobDefinition'or metadata.get('name')!=value['names'][action]or metadata.get('namespace')!=namespace or not metadata.get('uid'):raise ValueError('Use the actual owned IBM JobDefinition')
+        name=value['names'][action]+'-timer';current=get_resource(env,'pingsources.sources.knative.dev',name,namespace,optional=True)
+        if current is not None:ownership(current,job,namespace,prefix,name)
+        pending.append((action,job,name,current))
+    for action,job,name,current in pending:
+        spec=schedule_spec(job,action)
+        if install:
+            if current is None:
+                document={'apiVersion':'sources.knative.dev/v1','kind':'PingSource','metadata':{'name':name,'namespace':namespace,'labels':{'atmem.ai/operator-package':prefix},'ownerReferences':[{'apiVersion':'codeengine.cloud.ibm.com/v1beta1','kind':'JobDefinition','name':job['metadata']['name'],'uid':job['metadata']['uid'],'controller':False,'blockOwnerDeletion':False}]},'spec':spec}
+                kubectl(env,['create','-f','-','-o','json'],document)
+            elif any(current.get('spec',{}).get(key)!=child for key,child in spec.items()):
+                # Retain existing metadata/UID and resourceVersion. Native API
+                # conflicts fail; never --force, adopt or retry an overwrite.
+                document=copy.deepcopy(current);document.pop('status',None);document['spec']=spec
+                kubectl(env,['replace','-f','-','-o','json'],document)
+            kubectl(env,['wait','--for=condition=Ready','pingsources.sources.knative.dev/'+name,'--namespace',namespace,'--timeout=120s'])
+        actual=get_resource(env,'pingsources.sources.knative.dev',name,namespace)
+        ownership(actual,job,namespace,prefix,name);verify_ready(actual,spec)
+        result.append({'action':action,'name':name,'uid':actual['metadata']['uid'],'generation':actual['metadata']['generation'],'ready':True})
+    print(json.dumps({'format':'invoice-ibm-schedules-v1','sources':result,'jobExecutionValidated':False}))
+
+def main():
+    parser=argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('action',choices=['initialize','maintain','monitor','install-schedules','check-schedules']);parser.add_argument('--execute',action='store_true');args=parser.parse_args()
+    value=configuration();scheduling=args.action in {'install-schedules','check-schedules'}
+    if not args.execute:
+        plan=commands(value,'monitor')[:2]if scheduling else commands(value,args.action)
+        print(json.dumps({'execute':False,'commands':plan,'cloudOperations':False,'schedules':{action:{'job':value['names'][action],'cron':'17 2 * * *'if action=='maintain'else'7 * * * *','timezone':'UTC'}for action in ['maintain','monitor']}if scheduling else None}));return
+    with private_context()as env:
+        if scheduling:schedules(value,env,args.action=='install-schedules')
+        else:
+            for index,command in enumerate(commands(value,args.action)):subprocess.run(command,cwd=ROOT,env=env,check=True,timeout=1100 if index==2 else 120)
+
+if __name__=='__main__':main()
+`;

package/src/invoice-ibm.mjs

@@ -0,0 +1,142 @@
+import {invoiceBuilder,v,r,l,UUID}from'./invoice-builder.mjs';
+import {ibmRegistryHostExpression,ibmRegistryRegionValidation}from'./ibm-registry.mjs';
+import {ibmInvoiceOperations}from'./invoice-ibm-operations.mjs';
+import providerLock from'./runtime/invoice-ibm-provider-lock.json'with{type:'json'};
+
+// Native composition under qualification; no cloud operation is performed here.
+export function ibmInvoiceTerraform(project,row,environment='staging'){
+ if(row.provider!=='ibm')throw Error('IBM invoice compilation cannot substitute another cloud');
+ const b=invoiceBuilder(project,row,environment),{input,add,region}=b,n=row.address;
+ const account=input('account_id','Expected existing IBM Cloud account. No account is created.','a'.repeat(32),'^[0-9a-f]{32}$');
+ const group=input('resource_group_name','Existing reviewed destination resource group; never recreated.','reviewed-applications','^[A-Za-z0-9 _.-]{1,128}$');
+ const model=input('model_id','Reviewed regional watsonx JSON-object model; verify lifecycle, regional availability and processing terms.','ibm/reviewed-model','^[a-z0-9][a-z0-9_.-]{0,79}/[a-z0-9][a-z0-9_.-]{0,99}$');
+ const wxProject=input('watsonx_project_id','Existing watsonx project; authorize only the worker trusted profile and associate its runtime instance.','22222222-2222-2222-2222-222222222222',UUID);
+ const instance=input('runtime_instance_id','Existing associated watsonx.ai Runtime instance GUID, not a deployment ID.','33333333-3333-3333-3333-333333333333',UUID);
+ const endpoint=input('accounting_endpoint','Existing reviewed accounting HTTPS adapter; never provision a replacement.','https://accounting.example.test/actions','^https://[A-Za-z0-9.-]+(:[0-9]{1,5})?(/[^?#@ ]*)?$');
+ const scope=input('accounting_scope','Reviewed accounting scope bound to each approval.','owned-team','^[A-Za-z0-9_.:/-]{1,200}$');
+ const pull=input('registry_pull_secret','Name of the separately populated registry-pull secret in the new owned Code Engine project. No credential in Terraform.','registry-pull','^[a-z][a-z0-9-]{1,61}[a-z0-9]$');
+ if(!row.region)b.variable.region.validation=ibmRegistryRegionValidation();
+ for(const purpose of ['runtime','parser']){
+  input(purpose+'_image','Reviewed immutable image in this package-owned regional registry namespace. Empty during foundation.','au.icr.io/reviewed-namespace/'+purpose+'@sha256:'+'a'.repeat(64));b.variable[purpose+'_image'].default='';
+  b.variable[purpose+'_image'].validation={condition:'${!(var.applications_enabled'+(purpose==='runtime'?' || var.operations_enabled':'')+') || can(regex("@sha256:[0-9a-f]{64}$",var.'+purpose+'_image))}',error_message:'Use the reviewed immutable image digest before releasing applications.'};
+  b.resource.terraform_data.review_gate.lifecycle.precondition.push({condition:'${!(var.applications_enabled'+(purpose==='runtime'?' || var.operations_enabled':'')+') || startswith(var.'+purpose+'_image,format("%s/%s/'+purpose+'@sha256:",'+ibmRegistryHostExpression().slice(2,-1)+',local.name))}',error_message:'Images must use the exact owned regional registry namespace, not an invented hostname or foreign repository.'});
+  b.testExpressions[purpose+'_image']='format("au.icr.io/invoice-%s/'+purpose+'@sha256:'+'a'.repeat(64)+'",substr(sha256('+JSON.stringify(b.seed)+'),0,12))';
+ }
+ b.variable.staff_subjects={type:'set(string)',default:[],description:'Explicit existing admin-enrolled App ID subjects. Application membership/reviewer grants are separately admitted in SQL.',validation:{condition:'${alltrue([for subject in var.staff_subjects : can(regex("^[A-Za-z0-9_.:@-]{1,200}$",subject))])}',error_message:'Use exact verified staff subjects, never public signup or display-name guesses.'}};b.examples.staff_subjects=['44444444-4444-4444-4444-444444444444'];
+ b.variable.policy_review_base64={type:'string',default:'',description:'Scoped expiring model privacy/processing-terms review. Empty blocks inference; no approval is fabricated.'};b.examples.policy_review_base64='';
+ b.variable.replay_review_base64={type:'string',default:'',description:'Scoped expiring acceptance of actual accounting idempotency. Empty disables automatic uncertain replay.'};b.examples.replay_review_base64='';
+ b.data.ibm_resource_group={destination:{name:group}};
+ b.resource.terraform_data.review_gate.lifecycle.precondition.push({condition:'${data.ibm_resource_group.destination.account_id == var.account_id && (!var.applications_enabled || length(var.staff_subjects)>0)}',error_message:'Verify the destination account and explicit staff admission before release.'});
+ const rg=r('data.ibm_resource_group','destination','id'),projectId=r('ibm_code_engine_project',n,'project_id');
+ Object.assign(b.locals,{destination_contract:{id:'normalized-action-v1',endpointSha256:'${sha256(var.accounting_endpoint)}',scope,mappingVersion:1},destination_contract_base64:'${base64encode(jsonencode(local.destination_contract))}',database_config_name:l('name')+'-database',parser_secret_name:l('name')+'-parser',api_secret_name:l('name')+'-api-sql',worker_secret_name:l('name')+'-worker',bucket_name:'${format("%s-%s",local.name,substr(var.account_id,0,8))}'});
+ add('ibm_code_engine_project',n,{name:l('name'),resource_group_id:rg,lifecycle:{prevent_destroy:true}});
+ add('ibm_cr_namespace',n,{name:l('name'),resource_group_id:rg,lifecycle:{prevent_destroy:true}});
+ // Use the maintained Classic private-service endpoint contract, not an assumed
+ // direct Code Engine -> VPC/Gen2 route. There is no public SQL fallback.
+ add('ibm_database',n,{name:l('name'),service:'databases-for-postgresql',plan:'standard',location:region,resource_group_id:rg,version:'18',service_endpoints:'private',deletion_protection:true,group:{group_id:'member',memory:{allocation_mb:2048},disk:{allocation_mb:20480},members:{allocation_count:2}},configuration:JSON.stringify({log_statement:'none',log_min_error_statement:'panic',log_parameter_max_length_on_error:0}),lifecycle:{prevent_destroy:true}});
+ add('ibm_resource_instance',n+'_documents',{name:l('name')+'-documents',service:'cloud-object-storage',plan:'standard',location:'global',resource_group_id:rg,lifecycle:{prevent_destroy:true}});
+ add('ibm_cos_bucket',n,{bucket_name:l('bucket_name'),resource_instance_id:r('ibm_resource_instance',n+'_documents','id'),region_location:region,storage_class:'standard',endpoint_type:'public',force_delete:false,object_versioning:{enable:true},lifecycle:{prevent_destroy:true}});
+ add('ibm_cos_bucket_lifecycle_configuration',n,{bucket_crn:r('ibm_cos_bucket',n,'crn'),bucket_location:region,endpoint_type:'public',lifecycle_rule:[{rule_id:'invoice-originals-90-days',status:'enable',filter:{prefix:'accounts/'},expiration:{days:90},noncurrent_version_expiration:{noncurrent_days:1}},{rule_id:'unfinished-upload-cleanup',status:'enable',filter:{prefix:'accounts/'},abort_incomplete_multipart_upload:{days_after_initiation:1}}]});
+ for(const purpose of ['api','worker']){
+  add('ibm_iam_trusted_profile',n+'_'+purpose,{name:l('name')+'-'+purpose,description:'Dedicated invoice '+purpose+'; SQL login and business membership are independent.'});
+  add('ibm_iam_trusted_profile_link',n+'_'+purpose,{count:'${var.applications_enabled ? 1 : 0}',profile_id:r('ibm_iam_trusted_profile',n+'_'+purpose,'profile_id'),cr_type:'CE',link:{crn:r('ibm_code_engine_project',n,'crn'),component_name:l('name')+'-'+purpose,component_type:'app'},depends_on:['ibm_code_engine_app.'+n+'_'+purpose]});
+  add('ibm_iam_trusted_profile_policy',n+'_'+purpose+'_documents',{iam_id:r('ibm_iam_trusted_profile',n+'_'+purpose,'iam_id'),roles:purpose==='api'?['Object Reader','Object Writer']:['Object Reader'],resources:{service:'cloud-object-storage',resource_instance_id:r('ibm_resource_instance',n+'_documents','guid'),resource_type:'bucket',resource:l('bucket_name')}});
+ }
+ add('ibm_iam_trusted_profile_policy',n+'_model',{iam_id:r('ibm_iam_trusted_profile',n+'_worker','iam_id'),roles:['Writer'],resources:{service:'pm-20',resource_instance_id:instance,region},description:'Instance-scoped native watsonx role. Writer is broader than prediction-only; separate project collaborator admission remains required.'});
+ add('ibm_resource_instance',n+'_identity',{name:l('name')+'-staff',service:'appid',plan:'graduated-tier',location:region,resource_group_id:rg,lifecycle:{prevent_destroy:true}});
+ const tenant=r('ibm_resource_instance',n+'_identity','guid'),client=r('ibm_appid_application',n,'client_id');
+ add('ibm_appid_application',n,{tenant_id:tenant,name:l('name'),type:'singlepageapp'});
+ add('ibm_appid_application_scopes',n,{tenant_id:tenant,client_id:client,scopes:['process']});
+ add('ibm_appid_role',n,{tenant_id:tenant,name:'InvoiceStaff',access:[{application_id:client,scopes:['process']}],depends_on:['ibm_appid_application_scopes.'+n]});
+ add('ibm_appid_user_roles',n,{for_each:'${var.applications_enabled ? var.staff_subjects : toset([])}',tenant_id:tenant,subject:'${each.value}',role_ids:[r('ibm_appid_role',n,'role_id')]});
+ add('ibm_appid_idp_cloud_directory',n,{tenant_id:tenant,is_active:true,signup_enabled:false,self_service_enabled:false,welcome_enabled:false,reset_password_enabled:true,reset_password_notification_enabled:true,identity_field:'email',identity_confirm_access_mode:'FULL',identity_confirm_methods:['email']});
+ for(const idp of ['google','facebook','saml'])add('ibm_appid_idp_'+idp,n,{tenant_id:tenant,is_active:false});
+ const literal=(name,value)=>({name,value,type:'literal'}),reference=(name,type,key,ref)=>({name,type,key,reference:ref});
+ const common={ACCOUNT_ID:l('account_uuid'),REGION:region,WORKFLOW_NAMESPACE:'invoice',DATABASE_NAME:'ibmclouddb',DOCUMENT_BUCKET:l('bucket_name'),DESTINATION_CONTRACT_BASE64:l('destination_contract_base64')};
+ for(const purpose of ['api','worker','parser']){
+  const settings=purpose==='parser'?{PARSER_AUTH_MODE:'shared-token'}:purpose==='api'?{...common,SERVICE_NAME:l('name')+'-api',PUBLIC_ORIGIN_MODE:'code-engine-default',DATABASE_USER:l('api_db_user'),IAM_PROFILE_ID:r('ibm_iam_trusted_profile',n+'_api','profile_id'),APPID_TENANT_ID:tenant,APPID_CLIENT_ID:client,APPID_ISSUER:r('ibm_appid_application',n,'oauth_server_url'),AUTH_SCOPE:l('name')+'/process'}:{...common,DATABASE_USER:l('worker_db_user'),IAM_PROFILE_ID:r('ibm_iam_trusted_profile',n+'_worker','profile_id'),IBM_ACCOUNT_ID:account,WATSONX_PROJECT_ID:wxProject,RUNTIME_INSTANCE_ID:instance,MODEL_REGION:region,MODEL_ID:model,MAX_OUTPUT_TOKENS:'2048',POLICY_REVIEW_BASE64:v('policy_review_base64'),PARSER_ENDPOINT:r('ibm_code_engine_app',n+'_parser[0]','endpoint')+'/extract',DESTINATION_ENDPOINT:endpoint,DESTINATION_SCOPE:scope,DESTINATION_REPLAY_REVIEW_BASE64:v('replay_review_base64')};
+  const env=Object.entries(settings).map(([name,value])=>literal(name,value));
+  if(purpose!=='parser')env.push(...[['DATABASE_HOST','host'],['DATABASE_PORT','port'],['DATABASE_CA_PEM','ca_pem']].map(([name,key])=>reference(name,'config_map_key_reference',key,l('database_config_name'))),reference('DATABASE_PASSWORD','secret_key_reference','password',l(purpose==='api'?'api_secret_name':'worker_secret_name')));
+  if(purpose==='worker')env.push(reference('DESTINATION_TOKEN','secret_key_reference','accounting_token',l('worker_secret_name')));
+  if(purpose!=='api')env.push(reference('PARSER_SHARED_TOKEN','secret_key_reference','token',l('parser_secret_name')));
+  add('ibm_code_engine_app',n+'_'+purpose,{count:'${var.applications_enabled ? 1 : 0}',project_id:projectId,name:l('name')+'-'+purpose,image_reference:v(purpose==='parser'?'parser_image':'runtime_image'),image_secret:pull,image_port:8080,managed_domain_mappings:purpose==='api'?'local_public':purpose==='parser'?'local_private':'local',run_as_user:10001,run_service_account:'none',run_compute_resource_token_enabled:purpose!=='parser',scale_cpu_limit:'1',scale_memory_limit:purpose==='parser'?'2G':'1G',scale_min_instances:1,scale_max_instances:1,scale_concurrency:purpose==='api'?32:1,scale_concurrency_target:purpose==='api'?16:1,scale_request_timeout:purpose==='parser'?150:300,run_env_variables:env,...(purpose==='api'?{}:{run_commands:['python','-I','/app/'+(purpose==='worker'?'invoice_worker.py':'invoice_parser_service.py')]}),probe_readiness:{type:'http',path:'/healthz',port:8080,initial_delay:2,interval:5,timeout:2,failure_threshold:12},depends_on:purpose==='parser'?[]:['ibm_iam_trusted_profile_policy.'+n+'_'+purpose+'_documents',...(purpose==='worker'?['ibm_iam_trusted_profile_policy.'+n+'_model']:[])]});
+ }
+ add('ibm_appid_redirect_urls',n,{count:'${var.applications_enabled ? 1 : 0}',tenant_id:tenant,urls:[r('ibm_code_engine_app',n+'_api[0]','endpoint')+'/']});
+ b.output.application_url={value:'${var.applications_enabled ? ibm_code_engine_app.'+n+'_api[0].endpoint : null}'};
+ b.output.code_engine_project={value:projectId};b.output.registry={value:ibmRegistryHostExpression()+'/'+l('name')};
+ b.output.operator_deployment={value:{provider:'ibm',deploymentId:l('account_uuid'),accountId:l('account_uuid'),namespace:'invoice',tenantId:null,bindings:Object.fromEntries(['api','worker'].map(purpose=>[purpose,{name:l(purpose+'_db_user'),principal:'${uuidv5("url",'+JSON.stringify(b.seed+'/'+purpose+'-sql-credential')+')}'}]))}};
+ b.output.database_bootstrap={value:{deploymentCrn:r('ibm_database',n,'id'),database:'ibmclouddb',projectId,databaseConfig:l('database_config_name'),apiSecret:l('api_secret_name'),workerSecret:l('worker_secret_name'),parserSecret:l('parser_secret_name'),registryPullSecret:pull}};
+ ibmInvoiceOperations(b,n,{projectId,pull});
+ b.files['CODE_ENGINE_INPUTS.md']=IBM_INPUTS;
+ b.assertions.push(`run "private_database_and_parser" {\n command = plan\n assert {\n condition = ibm_database.${n}.service_endpoints == "private" && ibm_database.${n}.version == "18" && ibm_database.${n}.deletion_protection\n error_message = "Retain private native PostgreSQL and recovery protection."\n }\n assert {\n condition = ibm_code_engine_app.${n}_parser[0].managed_domain_mappings == "local_private" && !ibm_code_engine_app.${n}_parser[0].run_compute_resource_token_enabled && ibm_code_engine_app.${n}_parser[0].run_service_account == "none"\n error_message = "Untrusted PDF parser has private TLS and no cloud compute/API identity."\n }\n assert {\n condition = startswith(ibm_code_engine_app.${n}_api[0].image_reference,"au.icr.io/") && ibm_code_engine_app.${n}_worker[0].scale_min_instances == 1\n error_message = "Use the actual Sydney registry hostname and retain the background worker."\n }\n}\nrun "foundation_only" {\n command = plan\n variables {\n applications_enabled = false\n operations_enabled = false\n maintenance_enabled = false\n }\n assert {\n condition = length(ibm_code_engine_app.${n}_api) == 0 && length(ibm_code_engine_app.${n}_worker) == 0 && length(ibm_code_engine_app.${n}_parser) == 0 && length(ibm_code_engine_job.${n}_operator_initialize) == 0\n error_message = "Bootstrap and publication precede application release."\n }\n}\nrun "mutable_image_rejected" {\n command = plan\n variables { runtime_image = "unreviewed:latest" }\n expect_failures = [var.runtime_image]\n}\nrun "foreign_registry_rejected" {\n command = plan\n variables { runtime_image = "foreign.test/runtime@sha256:${'a'.repeat(64)}" }\n expect_failures = [terraform_data.review_gate]\n}\nrun "empty_staff_rejected" {\n command = plan\n variables { staff_subjects = [] }\n expect_failures = [terraform_data.review_gate]\n}\nrun "wrong_account_rejected" {\n command = plan\n variables { account_id = "${'c'.repeat(32)}" }\n expect_failures = [terraform_data.review_gate]\n}`);
+ let mock='mock_provider "ibm" {\n override_during = plan\n mock_data "ibm_resource_group" { defaults = { id="'+ 'b'.repeat(32)+'",account_id="'+'a'.repeat(32)+'" } }\n mock_resource "ibm_code_engine_project" { defaults = { project_id="11111111-1111-4111-8111-111111111111",crn="crn:v1:bluemix:public:codeengine:au-syd:a/'+ 'a'.repeat(32)+':11111111-1111-4111-8111-111111111111::" } }\n mock_resource "ibm_resource_instance" { defaults = { guid="55555555-5555-4555-8555-555555555555",id="crn:v1:bluemix:public:cloud-object-storage:global:a/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:55555555-5555-4555-8555-555555555555::" } }\n mock_resource "ibm_iam_trusted_profile" { defaults = { profile_id="Profile-66666666-6666-4666-8666-666666666666",iam_id="iam-Profile-66666666-6666-4666-8666-666666666666" } }\n mock_resource "ibm_appid_application" { defaults = { client_id="77777777-7777-4777-8777-777777777777",oauth_server_url="https://au-syd.appid.cloud.ibm.com/oauth/v4/55555555-5555-4555-8555-555555555555" } }\n}\n';
+ mock+='override_resource {\n override_during = plan\n target = ibm_cos_bucket.'+n+'\n values = { crn = "crn:v1:bluemix:public:cloud-object-storage:global:a/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:55555555-5555-4555-8555-555555555555:bucket:fixture-invoices" }\n}\n';
+ for(const [purpose,digit]of [['api','6'],['worker','8']])mock+='override_resource {\n override_during = plan\n target = ibm_iam_trusted_profile.'+n+'_'+purpose+'\n values = '+JSON.stringify({profile_id:'Profile-'+digit.repeat(8)+'-'+digit.repeat(4)+'-4'+digit.repeat(3)+'-8'+digit.repeat(3)+'-'+digit.repeat(12),iam_id:'iam-Profile-'+digit.repeat(8)+'-'+digit.repeat(4)+'-4'+digit.repeat(3)+'-8'+digit.repeat(3)+'-'+digit.repeat(12)})+'\n}\n';
+ for(const purpose of ['api','parser'])mock+='override_resource {\n override_during = plan\n target = ibm_code_engine_app.'+n+'_'+purpose+'[0]\n values = { endpoint = format("https://invoice-%s-'+purpose+'.project-fixture.'+(purpose==='parser'?'private.':'')+'au-syd.codeengine.appdomain.cloud",substr(sha256('+JSON.stringify(b.seed)+'),0,12)) }\n}\n';
+ return b.finish({ibm:{region}},{ibm:{source:'IBM-Cloud/ibm',version:providerLock.version}},providerLock.lock,mock,IBM_GUIDE);
+}
+const IBM_GUIDE=`# IBM invoice workflow (under qualification)
+
+A new owned Code Engine project hosts the public App ID/PKCE staff API, a minimum-one
+background queue worker and a separate private-TLS parser. Only worker/parser hold
+the parser-only 256-bit admission token; it grants no cloud/document/SQL/action
+permission. The parser has neither a compute resource token nor Code Engine API
+service-account authority. API and worker use separate native trusted profiles,
+SQL logins and secret references. Staff authentication is separate from live SQL
+membership and reviewer authority; public self-registration is disabled.
+
+The dedicated PostgreSQL 18 Classic instance uses private IBM service endpoints;
+no unimplemented direct VPC/Gen2 route or public SQL fallback is asserted. Exact
+regional service availability, private connectivity, CA verification and inherited
+permissions require authorized destination checks. Documents are private, versioned
+and stored regionally with a 90-day current / one-day noncurrent lifecycle. The
+regional S3 API uses authenticated TLS. Cloud control-plane/log/backup behavior and
+account policy need separate review; no exact erasure deadline is promised.
+
+Sydney Container Registry uses au.icr.io, not au-syd.icr.io. The namespace is owned
+by this package. Foundation creates resources with applications_enabled=false.
+CODE_ENGINE_INPUTS.md specifies separately populated config/secret references.
+Publish immutable reviewed images only when authorized, initialize SQL and staff
+membership through the separate operator, then review application release. The
+worker's instance-scoped watsonx Writer role is broader than prediction-only;
+separate project collaborator admission and the scoped model policy review remain
+required. An empty policy review blocks inference and never fabricates approval.
+
+Native schemas/mocks and local images do not prove the real account, IBM private
+network, model processing, restore or deployment. Full managed operator/scheduled
+retention delivery, monitoring, state strategy, registry/Spec Kit admission and the
+complete conversation/export checks remain qualification work. No cloud operation
+is performed by generating this package. Preserve the external accounting system.
+`;
+const IBM_INPUTS=`# Code Engine foundation inputs and ownership
+
+Use the database_bootstrap and operator_deployment outputs, not guessed identifiers.
+The new database's built-in application database is ibmclouddb; the separate operator
+owns the new workflow schema. Application SQL passwords are distinct strong random
+values created outside Terraform. Native trusted profiles authorize COS/model
+access, not database login: operator bindings use stable owned credential UUIDs,
+not a false claim of native IAM-to-SQL authentication.
+
+In the new Code Engine project, an authorized operator creates these references
+from private mode-0600 inputs, without placing secret values in Terraform/state:
+- registry_pull_secret: native registry configuration, private namespace pull only.
+- databaseConfig config map: exact private host, port, native PEM ca_pem and operator_user.
+- operator_jobs.operatorSecret: separate operator SQL password; only operating tasks.
+- apiSecret generic secret: password for the generated API SQL login only.
+- workerSecret generic secret: worker SQL password and existing accounting_token.
+- parserSecret generic secret: token, exactly 64 lowercase hexadecimal characters
+  from 32 cryptographically random bytes; only worker and parser reference it.
+
+Read native connection/CA metadata for this exact database deployment. Never use a
+public hostname or weaken TLS verification to bypass private connectivity. Supply
+bootstrap's private operator files and create both nonadministrative logins, exact
+schema/grants and memberships. Native CA/token rotation requires an explicit rollout;
+rotate parser/worker together and verify old/missing tokens are denied. The parent
+parser removes the token from its environment and its native tools use a minimal
+explicit environment. This is not an OS sandbox or proof of inherited platform IAM.
+
+The API derives its same-origin policy only from documented Code Engine app/domain/
+subdomain/region settings matched to its compiler-owned service name. No request
+Host or Forwarded header chooses a trusted origin. The App ID redirect resource
+references the created API endpoint after creation, avoiding a Terraform cycle.
+`;

package/src/invoice-operations.mjs

@@ -0,0 +1,143 @@
+export const INVOICE_OPERATIONS=`# Invoice workflow operations
+
+This is a private staff application. The generated API and worker SQL identities
+cannot create accounts, admit staff, erase audit history or perform operator repairs.
+Keep the schema-owner credential separate. No command below authorizes deployment.
+
+## Bootstrap and staff access
+
+Run Terraform foundation with applications_enabled=false. Review the actual account,
+regional availability, costs, private network access and provider identity settings.
+Supply the real accounting endpoint, scope and native secret outside Terraform state.
+Build, scan and publish the emitted API/worker and parser images when authorized;
+pin their exact digests in the emitted repositories before enabling applications.
+
+Create a private .local directory (mode 0700), and keep all operator input files mode
+0600 and outside source control. Run the operator from the emitted application
+directory, using a Python 3.13 environment installed with requirements.lock hashes.
+The operator needs an approved connection path to the private native database.
+Never expose PostgreSQL publicly to make bootstrap easier.
+
+operator-connection.json has exactly these fields:
+host, port (integer), dbname, user, passwordFile, caFile.
+The password/token and reviewed native PEM CA are absolute separate file paths.
+There is no inline password, insecure TLS option or arbitrary connection string.
+AWS includes rds-global-ca.pem plus its pinned public-source receipt; verify native
+CA availability for the selected database and update/rebuild before CA expiry.
+IBM supplies its database-specific CA and separately provisioned SQL passwords.
+Azure's operator must be authorized for native Entra mapping in postgres as well as
+schema administration in the application database. Google native IAM SQL users must
+exist before the separate schema/grant step. Short-lived tokens must remain valid
+for the bounded operator action; the tool does not silently acquire extra privilege.
+
+operator-deployment.json has exactly: provider, deploymentId (UUID), accountId (UUID),
+namespace (invoice), bindings (api and worker), tenantId (Azure UUID, otherwise null).
+Each binding has name and principal; Google additionally has created_by_package:true.
+Use exact Terraform outputs/native identity subjects, never guessed display names.
+AWS emits the complete nonsecret operator_deployment output. With an approved
+foundation and private connection available, retain that output in the private file.
+
+Preview (no database connection or write):
+
+    python -I invoice_operator.py initialize --connection-file .local/operator-connection.json --deployment-file .local/operator-deployment.json
+
+Add --execute only for the reviewed authorized destination. Initialization creates
+owned native login mappings, the exact version-1 schema/queue and restricted grants.
+It refuses to adopt an unrelated role or schema. Re-running the exact owned bootstrap
+is supported; a changed schema requires a reviewed migration, not reinitialization.
+IBM initialization additionally reads --input-file with apiPasswordFile and
+workerPasswordFile pointing to private strong passwords; never rotate existing
+owned passwords implicitly. Roll out credentials explicitly when rotating them.
+
+Membership uses --input-file containing subject (verified native login subject),
+roles (explicit array of user/reviewer/supervisor/owner), active (boolean):
+
+    python -I invoice_operator.py membership --connection-file .local/operator-connection.json --deployment-file .local/operator-deployment.json --input-file .local/member.json --execute
+
+Inviting/signing in at the native identity service does not automatically grant
+application membership. Ordinary staff get user; three designated reviewers get
+reviewer. Revocation uses active:false; role replacement is explicit. Native JWT
+expiry and live SQL membership are different boundaries. Test both at the destination.
+
+## Ninety-day retention
+
+The invoice API and processing stages refuse payloads whose creation time is outside
+the 90-day retention window, including data recovered from old backups. Private-store
+lifecycles remove original objects separately; version/noncurrent deletion and native
+provider deletion are asynchronous. Backups may retain expired data for the stated
+backup window. This is not an exact physical-deletion deadline or legal-retention SLA.
+
+AWS, Azure and Google emit fixed operating jobs/schedules in OPERATING_JOBS.md.
+The IBM native job adapter remains under qualification; do not assume its schedules
+exist. The shared job entrypoint supports preview (without --execute), initialize,
+maintain and monitor using the emitted owned metadata and separate operator identity.
+It is never an API/model endpoint.
+
+For the manual CLI maintenance path, pause uploads/API and workers, allow bounded
+work to settle, then run:
+
+    python -I invoice_operator.py purge-expired --connection-file .local/operator-connection.json --deployment-file .local/operator-deployment.json --execute --confirm-workloads-stopped
+
+It removes at most 500 expired operations, their historical suggestions/audit and
+owned queue notifications per transaction. Repeat while moreMayRemain is true.
+Active queue jobs/live leases cause rollback; settle them before retrying. Other
+accounts/namespaces, staff memberships and young records are preserved. Retain the
+count-only maintenance receipt and alert on failure/overdue runs. The fixed
+maintain job additionally bounds a whole execution to twenty 500-record batches,
+reports remaining backlog nonzero and uses the same live-job/lease/ownership guards
+for online expired-only cleanup. Concurrent young submissions are serialized by
+transaction locks and preserved. Restore/reconciliation still require the isolated
+paused boundary. See the provider-specific native job definitions for scheduling;
+a script file alone does not establish that a schedule exists.
+
+Deduplication is scoped to the retained operation lifetime. Do not promise unlimited
+historical deduplication after the client-directed 90-day payload/audit deletion.
+Destination operators must independently retain the idempotency identity as required
+by their accounting contract. Do not clear an unknown outcome to resend it.
+
+## Backup, restore and uncertain deliveries
+
+Review native managed backup retention, encryption, restore region and current
+provider procedures. Restore to an isolated new database/endpoint with API/workers
+stopped. Do not point running workers at a restored snapshot: the accounting service
+may have accepted deliveries after that snapshot's recovery point.
+
+With private operator files explicitly pointing to the isolated restored database:
+
+    python -I invoice_operator.py quarantine-restore --connection-file .local/restored-connection.json --deployment-file .local/operator-deployment.json --execute --confirm-workloads-stopped
+
+Repeat while moreMayRemain is true. The tool fences restored approved/in-flight
+deliveries as unknown, preserves approval hashes and clears stale queue notifications.
+Interrupted extraction leases are fenced back to pending; the normal worker rebuilds
+those notifications after expiry cleanup. This avoids retaining a restored doing
+job forever or restarting an old external delivery. Attempt limits are preserved.
+The stopped-workloads flag is an operator assertion, not proof that cloud workloads
+were stopped. Purge expired restored records before reopening. Reapply reviewed native
+runtime identity grants/membership as appropriate; verify current revocations and
+object versions. Exercise read, denied access, fresh extraction and same-key behavior
+on an isolated synthetic fixture before cutover. Keep old endpoints for rollback;
+never dual-run workers on divergent databases. A local logical restore test does not
+prove native PITR, object recovery, production RPO/RTO or destination availability.
+
+For each uncertain delivery, independently query the existing accounting system by
+operation ID and payload digest. If it confirms completion, store a private JSON
+receipt with operation_id, payload_sha256, external_id and accountable operator:
+
+    python -I invoice_operator.py reconcile-completed --connection-file .local/operator-connection.json --deployment-file .local/operator-deployment.json --input-file .local/verified-receipt.json --execute --confirm-workloads-stopped
+
+The tool checks exact ownership/state/approval integrity and records the supplied
+receipt without making an external request. It does not verify the human's accounting
+lookup. A missing receipt is not permission to resend. Automatic same-key replay needs
+the scoped, expiring destination-contract review described in the worker configuration.
+Never rewrite an approved payload, endpoint, account or idempotency key for a retry.
+
+## Monitoring and release checks
+
+Alert on stale pending/approved work, dead/unknown outcomes, repeated parser/model
+failures, storage/database growth and failed retention windows. Healthz means process
+alive; it does not establish database/model/destination readiness. Keep payloads,
+credentials and document text out of telemetry. Scale worker/parser capacity together
+after throughput/quota tests. Before production, qualify native sign-in/token expiry,
+workload IAM, private ingress, TLS, native storage versions, actual model quality,
+external accounting idempotency and restore/cutover with authorized destination tests.
+`;