@sasjs/utils 2.49.0 → 2.49.2

package/macros/mp_cleancsv.sas

@@ -0,0 +1,78 @@
+/**
+  @file
+  @brief Fixes embedded cr / lf / crlf in CSV
+  @details CSVs will sometimes contain lf or crlf within quotes (eg when
+  saved by excel).  When the termstr is ALSO lf or crlf that can be tricky
+  to process using SAS defaults.
+  This macro converts any csv to follow the convention of a windows excel file,
+  applying CRLF line endings and converting embedded cr and crlf to lf.
+
+  Usage:
+
+      fileref mycsv "/path/your/csv";
+      %mp_cleancsv(in=mycsv,out=/path/new.csv)
+
+  @param in= (NOTPROVIDED) Provide path or fileref to input csv. If a period is
+    found, it is assumed to be a file.
+  @param out= (NOTPROVIDED) Output path or fileref to output csv.  If a period
+    is found, it is assumed to be a file.
+  @param qchar= ('22'x) Quote char - hex code 22 is the double quote.
+
+  @version 9.2
+  @author Allan Bowe
+  @cond
+**/
+
+%macro mp_cleancsv(in=NOTPROVIDED,out=NOTPROVIDED,qchar='22'x);
+%if "&in"="NOTPROVIDED" or "&out"="NOTPROVIDED" %then %do;
+  %put %str(ERR)OR: Please provide valid input (&in) & output (&out) locations;
+  %return;
+%end;
+
+/* presence of a period(.) indicates a physical location */
+%if %index(&in,.) %then %let in="&in";
+%if %index(&out,.) %then %let out="&out";
+
+/**
+  * convert all cr and crlf within quotes to lf
+  * convert all other cr or lf to crlf
+  */
+  data _null_;
+    infile &in recfm=n ;
+    file &out recfm=n;
+    retain isq iscrlf 0 qchar &qchar;
+    input inchar $char1. ;
+    if inchar=qchar then isq = mod(isq+1,2);
+    if isq then do;
+      /* inside a quote change cr and crlf to lf */
+      if inchar='0D'x then do;
+        put '0A'x;
+        input inchar $char1.;
+        if inchar ne '0A'x then do;
+          put inchar $char1.;
+          if inchar=qchar then isq = mod(isq+1,2);
+        end;
+      end;
+      else put inchar $char1.;
+    end;
+    else do;
+      /* outside a quote, change cr and lf to crlf */
+      if inchar='0D'x then do;
+        crblank:
+        put '0D0A'x;
+        input inchar $char1.;
+        if inchar='0D'x then do;
+          /* multiple CR indicates CR formatted file with blank lines */
+          goto crblank;
+        end;
+        else if inchar ne '0A'x then do;
+          put inchar $char1.;
+          if inchar=qchar then isq = mod(isq+1,2);
+        end;
+      end;
+      else if inchar='0A'x then put '0D0A'x;
+      else put inchar $char1.;
+    end;
+  run;
+%mend mp_cleancsv;
+/** @endcond */

package/macros/mp_cntlout.sas

@@ -0,0 +1,85 @@
+/**
+  @file mp_cntlout.sas
+  @brief Creates a cntlout dataset in a consistent format
+  @details The dataset produced by proc format in the cntlout option will vary
+  according to its contents.
+
+  When dealing with formats from an ETL perspective (eg in [Data Controller for
+  SAS](https://datacontroller.io)), it is important that the output dataset
+  has a consistent model (and compariable values).
+
+  This macro makes use of mddl_sas_cntlout.sas to provide the consistent model,
+  and will left-align the start and end values when dealing with numeric ranges
+  to enable consistency when checking for differences.
+
+  usage:
+
+      %mp_cntlout(libcat=yourlib.cat,cntlout=work.formatexport)
+
+  @param [in] libcat The library.catalog reference
+  @param [in] fmtlist= (0) provide a space separated list of specific formats to
+    extract
+  @param [in] iftrue= (1=1) A condition under which the macro should be executed
+  @param [out] cntlout= (work.fmtextract) Libds reference for the output dataset
+
+  <h4> SAS Macros </h4>
+  @li mddl_sas_cntlout.sas
+  @li mf_getuniquename.sas
+
+  <h4> Related Macros </h4>
+  @li mf_getvarformat.sas
+  @li mp_getformats.sas
+  @li mp_loadformat.sas
+  @li mp_ds2fmtds.sas
+
+  @version 9.2
+  @author Allan Bowe
+  @cond
+**/
+
+%macro mp_cntlout(
+  iftrue=(1=1)
+  ,libcat=
+  ,cntlout=work.fmtextract
+  ,fmtlist=0
+)/*/STORE SOURCE*/;
+%local ddlds cntlds i;
+
+%if not(%eval(%unquote(&iftrue))) %then %return;
+
+%let ddlds=%mf_getuniquename();
+%let cntlds=%mf_getuniquename();
+
+%mddl_sas_cntlout(libds=&ddlds)
+
+%if %index(&libcat,-)>0 and %scan(&libcat,2,-)=FC %then %do;
+  %let libcat=%scan(&libcat,1,-);
+%end;
+
+proc format lib=&libcat cntlout=&cntlds;
+%if "&fmtlist" ne "0" %then %do;
+  select
+  %do i=1 %to %sysfunc(countw(&fmtlist));
+    %scan(&fmtlist,&i,%str( ))
+  %end;
+  ;
+%end;
+run;
+
+data &cntlout;
+  if 0 then set &ddlds;
+  set &cntlds;
+  if type="N" then do;
+    start=cats(start);
+    end=cats(end);
+  end;
+run;
+proc sort;
+  by fmtname start;
+run;
+
+proc sql;
+drop table &ddlds,&cntlds;
+
+%mend mp_cntlout;
+/** @endcond */

package/macros/mp_copyfolder.sas

@@ -0,0 +1,84 @@
+/**
+  @file
+  @brief A macro to recursively copy a directory
+  @details Performs a recursive directory listing then works from top to bottom
+    copying files and creating subdirectories.
+
+  Usage:
+
+      %let rootdir=%sysfunc(pathname(work))/demo;
+      %let copydir=%sysfunc(pathname(work))/demo_copy;
+      %mf_mkdir(&rootdir)
+      %mf_mkdir(&rootdir/subdir)
+      %mf_mkdir(&rootdir/subdir/subsubdir)
+      data "&rootdir/subdir/example.sas7bdat";
+      run;
+
+      %mp_copyfolder(&rootdir,&copydir)
+
+  @param [in] source Unquoted path to the folder to copy from.
+  @param [out] target Unquoted path to the folder to copy to.
+  @param [in] copymax=(MAX) Set to a positive integer to indicate the level of
+    subdirectory copy recursion - eg 3, to go `./3/levels/deep`.  For unlimited
+    recursion, set to MAX.
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquename.sas
+  @li mf_isdir.sas
+  @li mf_mkdir.sas
+  @li mp_abort.sas
+  @li mp_dirlist.sas
+
+  <h4> Related Macros </h4>
+  @li mp_copyfolder.test.sas
+
+**/
+
+%macro mp_copyfolder(source,target,copymax=MAX);
+
+  %mp_abort(iftrue=(%mf_isdir(&source)=0)
+    ,mac=&sysmacroname
+    ,msg=%str(Source dir does not exist (&source))
+  )
+
+  %mf_mkdir(&target)
+
+  %mp_abort(iftrue=(%mf_isdir(&target)=0)
+    ,mac=&sysmacroname
+    ,msg=%str(Target dir could not be created (&target))
+  )
+
+  /* prep temp table */
+  %local tempds;
+  %let tempds=%mf_getuniquename();
+
+  /* recursive directory listing */
+  %mp_dirlist(path=&source,outds=work.&tempds,maxdepth=&copymax)
+
+  /* create folders and copy content */
+  data _null_;
+    length msg $200;
+    call missing(msg);
+    set work.&tempds;
+    if _n_ = 1 then dpos+sum(length(directory),2);
+    filepath2="&target/"!!substr(filepath,dpos);
+    if file_or_folder='folder' then call execute('%mf_mkdir('!!filepath2!!')');
+    else do;
+      length fref1 fref2 $8;
+      rc1=filename(fref1,filepath,'disk','recfm=n');
+      rc2=filename(fref2,filepath2,'disk','recfm=n');
+      if fcopy(fref1,fref2) ne 0 then do;
+        msg=sysmsg();
+        putlog "%str(ERR)OR: Unable to copy " filepath " to " filepath2;
+        putlog msg=;
+      end;
+    end;
+    rc=filename(fref1);
+    rc=filename(fref2);
+  run;
+
+  /* tidy up */
+  proc sql;
+  drop table work.&tempds;
+
+%mend mp_copyfolder;

package/macros/mp_coretable.sas

@@ -0,0 +1,73 @@
+/**
+  @file
+  @brief Create the permanent Core tables
+  @details Several macros in the [core](https://github.com/sasjs/core) library
+    make use of permanent tables.  To avoid duplication in definitions, this
+    macro provides a central location for managing the corresponding DDL.
+
+  Note - this macro is likely to be deprecated in future in favour of a
+  dedicated "datamodel" folder (prefix mddl)
+
+  Any corresponding data would go in a seperate repo, to avoid this one
+  ballooning in size!
+
+  Example usage:
+
+      %mp_coretable(LOCKTABLE,libds=work.locktable)
+
+  @param [in] table_ref The type of table to create.  Example values:
+    @li DIFFTABLE
+    @li FILTER_DETAIL
+    @li FILTER_SUMMARY
+    @li LOCKANYTABLE
+    @li MAXKEYTABLE
+  @param [in] libds= (0) The library.dataset reference used to create the table.
+    If not provided, then the DDL is simply printed to the log.
+
+  <h4> SAS Macros </h4>
+  @li mddl_dc_difftable.sas
+  @li mddl_dc_filterdetail.sas
+  @li mddl_dc_filtersummary.sas
+  @li mddl_dc_locktable.sas
+  @li mddl_dc_maxkeytable.sas
+  @li mf_getuniquename.sas
+
+  <h4> Related Macros </h4>
+  @li mp_filterstore.sas
+  @li mp_lockanytable.sas
+  @li mp_retainedkey.sas
+  @li mp_storediffs.sas
+  @li mp_stackdiffs.sas
+
+  @version 9.2
+  @author Allan Bowe
+
+**/
+
+%macro mp_coretable(table_ref,libds=0
+)/*/STORE SOURCE*/;
+%local outds ;
+%let outds=%sysfunc(ifc(&libds=0,%mf_getuniquename(),&libds));
+proc sql;
+%if &table_ref=DIFFTABLE %then %do;
+  %mddl_dc_difftable(libds=&outds)
+%end;
+%else %if &table_ref=LOCKTABLE %then %do;
+  %mddl_dc_locktable(libds=&outds)
+%end;
+%else %if &table_ref=FILTER_SUMMARY %then %do;
+  %mddl_dc_filtersummary(libds=&outds)
+%end;
+%else %if &table_ref=FILTER_DETAIL %then %do;
+  %mddl_dc_filterdetail(libds=&outds)
+%end;
+%else %if &table_ref=MAXKEYTABLE %then %do;
+  %mddl_dc_maxkeytable(libds=&outds)
+%end;
+
+%if &libds=0 %then %do;
+  proc sql;
+  describe table &syslast;
+  drop table &syslast;
+%end;
+%mend mp_coretable;

package/macros/mp_createconstraints.sas

@@ -0,0 +1,70 @@
+/**
+  @file mp_createconstraints.sas
+  @brief Creates constraints
+  @details Takes the output from mp_getconstraints.sas as input
+
+        proc sql;
+        create table work.example(
+          TX_FROM float format=datetime19.,
+          DD_TYPE char(16),
+          DD_SOURCE char(2048),
+          DD_SHORTDESC char(256),
+          constraint pk primary key(tx_from, dd_type,dd_source),
+          constraint unq unique(tx_from, dd_type),
+          constraint nnn not null(DD_SHORTDESC)
+        );
+
+      %mp_getconstraints(lib=work,ds=example,outds=work.constraints)
+      %mp_deleteconstraints(inds=work.constraints,outds=dropped,execute=YES)
+      %mp_createconstraints(inds=work.constraints,outds=created,execute=YES)
+
+  @param inds= (work.mp_getconstraints) The input table containing the
+    constraint info
+  @param outds= (work.mp_createconstraints) A table containing the create
+    statements (create_statement column)
+  @param execute= (NO) To actually create, use YES.
+
+  <h4> Related Files </h4>
+  @li mp_getconstraints.sas
+
+  @version 9.2
+  @author Allan Bowe
+
+**/
+
+%macro mp_createconstraints(inds=mp_getconstraints
+  ,outds=work.mp_createconstraints
+  ,execute=NO
+)/*/STORE SOURCE*/;
+
+proc sort data=&inds out=&outds;
+  by libref table_name constraint_name;
+run;
+
+data &outds;
+  set &outds;
+  by libref table_name constraint_name;
+  length create_statement $500;
+  if _n_=1 and "&execute"="YES" then call execute('proc sql;');
+  if first.constraint_name then do;
+    if constraint_type='PRIMARY' then type='PRIMARY KEY';
+    else type=constraint_type;
+    create_statement=catx(" ","alter table",libref,".",table_name
+      ,"add constraint",constraint_name,type,"(");
+    if last.constraint_name then
+      create_statement=cats(create_statement,column_name,");");
+    else create_statement=cats(create_statement,column_name,",");
+    if "&execute"="YES" then call execute(create_statement);
+  end;
+  else if last.constraint_name then do;
+    create_statement=cats(column_name,");");
+    if "&execute"="YES" then call execute(create_statement);
+  end;
+  else do;
+    create_statement=cats(column_name,",");
+    if "&execute"="YES" then call execute(create_statement);
+  end;
+  output;
+run;
+
+%mend mp_createconstraints;

package/macros/mp_createwebservice.sas

@@ -0,0 +1,32 @@
+/**
+  @file mp_createwebservice.sas
+  @brief Create a web service in SAS 9, Viya or SASjs Server
+  @details This is actually a wrapper for mx_createwebservice.sas, remaining
+  for legacy purposes.  For new apps, use mx_createwebservice.sas.
+
+
+  <h4> SAS Macros </h4>
+  @li mx_createwebservice.sas
+
+
+**/
+
+%macro mp_createwebservice(path=HOME
+    ,name=initService
+    ,precode=
+    ,code=ft15f001
+    ,desc=This service was created by the mp_createwebservice macro
+    ,replace=YES
+    ,mdebug=0
+)/*/STORE SOURCE*/;
+
+  %mx_createwebservice(path=&path
+    ,name=&name
+    ,precode=&precode
+    ,code=&code
+    ,desc=&desc
+    ,replace=&replace
+    ,mdebug=&mdebug
+  )
+
+%mend mp_createwebservice;

package/macros/mp_csv2ds.sas

@@ -0,0 +1,140 @@
+/**
+  @file mp_csv2ds.sas
+  @brief Efficient import of arbitrary CSV using a dataset as template
+  @details Used to import relevant columns from a large CSV using
+  a dataset to provide the types and lengths.  Assumes that a header
+  row is provided, and datarows start on line 2.  Extra columns in
+  both the CSV and base dataset are ignored.
+
+  Usage:
+
+      filename mycsv temp;
+      data _null_;
+        file mycsv;
+        put 'name,age,nickname';
+        put 'John,48,Jonny';
+        put 'Jennifer,23,Jen';
+      run;
+
+      %mp_csv2ds(inref=mycsv,outds=myds,baseds=sashelp.class)
+
+
+  @param inref= fileref to the CSV
+  @param outds= output ds (lib.ds format)
+  @param view= Set to YES or NO to determine whether the output should be
+    a view or not.  Default is NO (not a view).
+  @param baseds= Template dataset on which to create the input statement.
+    Is used to determine types, lengths, and any informats.
+
+  @version 9.2
+  @author Allan Bowe
+
+  <h4> SAS Macros </h4>
+  @li mp_abort.sas
+  @li mf_existds.sas
+
+**/
+
+%macro mp_csv2ds(inref=0,outds=0,baseds=0,view=NO);
+
+%mp_abort(iftrue=( &inref=0 )
+  ,mac=&sysmacroname
+  ,msg=%str(the INREF variable must be provided)
+)
+%mp_abort(iftrue=( %superq(outds)=0 )
+  ,mac=&sysmacroname
+  ,msg=%str(the OUTDS variable must be provided)
+)
+%mp_abort(iftrue=( &baseds=0 )
+  ,mac=&sysmacroname
+  ,msg=%str(the BASEDS variable must be provided)
+)
+%mp_abort(iftrue=( %mf_existds(&baseds)=0 )
+  ,mac=&sysmacroname
+  ,msg=%str(the BASEDS dataset (&baseds) needs to be assigned, and to exist)
+)
+
+/* count rows */
+%local hasheader; %let hasheader=0;
+data _null_;
+  if _N_ > 1 then do;
+    call symputx('hasheader',1,'l');
+    stop;
+  end;
+  infile &inref;
+  input;
+run;
+%mp_abort(iftrue=( &hasheader=0 )
+  ,mac=&sysmacroname
+  ,msg=%str(No header row in &inref)
+)
+
+/* get the variables in the CSV */
+data _data_;
+  infile &inref;
+  input;
+  length name $32;
+  do i=1 to countc(_infile_,',')+1;
+    name=upcase(scan(_infile_,i,','));
+    output;
+  end;
+  stop;
+run;
+%local csv_vars;%let csv_vars=&syslast;
+
+/* get the variables in the dataset */
+proc contents noprint data=&baseds
+  out=_data_ (keep=name type length format: informat);
+run;
+%local base_vars; %let base_vars=&syslast;
+
+proc sql undo_policy=none;
+create table &csv_vars as
+  select a.*
+    ,b.type
+    ,b.length
+    ,b.format
+    ,b.formatd
+    ,b.formatl
+    ,b.informat
+  from &csv_vars a
+  left join &base_vars b
+  on a.name=upcase(b.name)
+  order by i;
+
+/* prepare the input statement */
+%local instat dropvars;
+data _null_;
+  set &syslast end=last;
+  length in dropvars $32767;
+  retain in dropvars;
+  if missing(type) then do;
+    informat='$1.';
+    dropvars=catx(' ',dropvars,name);
+  end;
+  else if missing(informat) then do;
+    if type=1 then informat='best.';
+    else informat=cats('$',length,'.');
+  end;
+  else informat=cats(informat,'.');
+  in=catx(' ',in,name,':',informat);
+  if last then do;
+    call symputx('instat',in,'l');
+    call symputx('dropvars',dropvars,'l');
+  end;
+run;
+
+/* import the CSV */
+data &outds
+  %if %upcase(&view)=YES %then %do;
+    /view=&outds
+  %end;
+  ;
+  infile &inref dsd firstobs=2;
+  input &instat;
+  %if %length(&dropvars)>0 %then %do;
+    drop &dropvars;
+  %end;
+run;
+
+%mend mp_csv2ds;

package/macros/mp_deleteconstraints.sas

@@ -0,0 +1,52 @@
+/**
+  @file mp_deleteconstraints.sas
+  @brief Delete constraionts
+  @details Takes the output from mp_getconstraints.sas as input
+
+        proc sql;
+        create table work.example(
+          TX_FROM float format=datetime19.,
+          DD_TYPE char(16),
+          DD_SOURCE char(2048),
+          DD_SHORTDESC char(256),
+          constraint pk primary key(tx_from, dd_type,dd_source),
+          constraint unq unique(tx_from, dd_type),
+          constraint nnn not null(DD_SHORTDESC)
+        );
+
+      %mp_getconstraints(lib=work,ds=example,outds=work.constraints)
+      %mp_deleteconstraints(inds=work.constraints,outds=dropped,execute=YES)
+
+  @param inds= The input table containing the constraint info
+  @param outds= a table containing the drop statements (drop_statement column)
+  @param execute= `YES|NO` - default is NO. To actually drop, use YES.
+
+
+  @version 9.2
+  @author Allan Bowe
+
+**/
+
+%macro mp_deleteconstraints(inds=mp_getconstraints
+  ,outds=mp_deleteconstraints
+  ,execute=NO
+)/*/STORE SOURCE*/;
+
+proc sort data=&inds out=&outds;
+  by libref table_name constraint_name;
+run;
+
+data &outds;
+  set &outds;
+  by libref table_name constraint_name;
+  length drop_statement $500;
+  if _n_=1 and "&execute"="YES" then call execute('proc sql;');
+  if first.constraint_name then do;
+    drop_statement=catx(" ","alter table",libref,".",table_name
+      ,"drop constraint",constraint_name,";");
+    output;
+    if "&execute"="YES" then call execute(drop_statement);
+  end;
+run;
+
+%mend mp_deleteconstraints;

package/macros/mp_deletefolder.sas

@@ -0,0 +1,70 @@
+/**
+  @file
+  @brief A macro to delete a directory
+  @details Will delete all folder content (including subfolder content) and
+    finally, the folder itself.
+
+  Usage:
+
+      %let rootdir=%sysfunc(pathname(work))/demo;
+      %mf_mkdir(&rootdir)
+      %mf_mkdir(&rootdir/subdir)
+      %mf_mkdir(&rootdir/subdir/subsubdir)
+      data "&rootdir/subdir/example.sas7bdat";
+      run;
+
+      %mp_deletefolder(&rootdir)
+
+  @param path Unquoted path to the folder to delete.
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquename.sas
+  @li mf_isdir.sas
+  @li mp_dirlist.sas
+
+  <h4> Related Macros </h4>
+  @li mp_deletefolder.test.sas
+
+**/
+
+%macro mp_deletefolder(folder);
+  /* proceed if valid directory */
+  %if %mf_isdir(&folder)=1 %then %do;
+
+    /* prep temp table */
+    %local tempds;
+    %let tempds=%mf_getuniquename();
+
+    /* recursive directory listing */
+    %mp_dirlist(path=&folder,outds=work.&tempds, maxdepth=MAX)
+
+    /* sort descending level so can delete folder contents before folders */
+    proc sort data=work.&tempds;
+      by descending level;
+    run;
+
+    /* ensure top level folder is removed at the end */
+    proc sql;
+    insert into work.&tempds set filepath="&folder";
+
+    /* delete everything */
+    data _null_;
+      set work.&tempds end=last;
+      length fref $8;
+      fref='';
+      rc=filename(fref,filepath);
+      rc=fdelete(fref);
+      if rc then do;
+        msg=sysmsg();
+        put "&sysmacroname:" / rc= / msg= / filepath=;
+      end;
+      rc=filename(fref);
+    run;
+
+    /* tidy up */
+    proc sql;
+    drop table work.&tempds;
+
+  %end;
+  %else %put &sysmacroname: &folder: is not a valid / accessible folder. ;
+%mend mp_deletefolder;