rubyfox-server 2.3.0.pre2

data/lib/rubyfox/server/data/lib/Lib/pipes.py

@@ -0,0 +1,297 @@
+"""Conversion pipeline templates.
+
+The problem:
+------------
+
+Suppose you have some data that you want to convert to another format,
+such as from GIF image format to PPM image format.  Maybe the
+conversion involves several steps (e.g. piping it through compress or
+uuencode).  Some of the conversion steps may require that their input
+is a disk file, others may be able to read standard input; similar for
+their output.  The input to the entire conversion may also be read
+from a disk file or from an open file, and similar for its output.
+
+The module lets you construct a pipeline template by sticking one or
+more conversion steps together.  It will take care of creating and
+removing temporary files if they are necessary to hold intermediate
+data.  You can then use the template to do conversions from many
+different sources to many different destinations.  The temporary
+file names used are different each time the template is used.
+
+The templates are objects so you can create templates for many
+different conversion steps and store them in a dictionary, for
+instance.
+
+
+Directions:
+-----------
+
+To create a template:
+    t = Template()
+
+To add a conversion step to a template:
+   t.append(command, kind)
+where kind is a string of two characters: the first is '-' if the
+command reads its standard input or 'f' if it requires a file; the
+second likewise for the output. The command must be valid /bin/sh
+syntax.  If input or output files are required, they are passed as
+$IN and $OUT; otherwise, it must be  possible to use the command in
+a pipeline.
+
+To add a conversion step at the beginning:
+   t.prepend(command, kind)
+
+To convert a file to another file using a template:
+  sts = t.copy(infile, outfile)
+If infile or outfile are the empty string, standard input is read or
+standard output is written, respectively.  The return value is the
+exit status of the conversion pipeline.
+
+To open a file for reading or writing through a conversion pipeline:
+   fp = t.open(file, mode)
+where mode is 'r' to read the file, or 'w' to write it -- just like
+for the built-in function open() or for os.popen().
+
+To create a new template object initialized to a given one:
+   t2 = t.clone()
+
+For an example, see the function test() at the end of the file.
+"""                                     # '
+
+
+import re
+
+import os
+import tempfile
+import string
+
+__all__ = ["Template"]
+
+# Conversion step kinds
+
+FILEIN_FILEOUT = 'ff'                   # Must read & write real files
+STDIN_FILEOUT  = '-f'                   # Must write a real file
+FILEIN_STDOUT  = 'f-'                   # Must read a real file
+STDIN_STDOUT   = '--'                   # Normal pipeline element
+SOURCE         = '.-'                   # Must be first, writes stdout
+SINK           = '-.'                   # Must be last, reads stdin
+
+stepkinds = [FILEIN_FILEOUT, STDIN_FILEOUT, FILEIN_STDOUT, STDIN_STDOUT, \
+             SOURCE, SINK]
+
+
+class Template:
+    """Class representing a pipeline template."""
+
+    def __init__(self):
+        """Template() returns a fresh pipeline template."""
+        self.debugging = 0
+        self.reset()
+
+    def __repr__(self):
+        """t.__repr__() implements `t`."""
+        return '<Template instance, steps=' + `self.steps` + '>'
+
+    def reset(self):
+        """t.reset() restores a pipeline template to its initial state."""
+        self.steps = []
+
+    def clone(self):
+        """t.clone() returns a new pipeline template with identical
+        initial state as the current one."""
+        t = Template()
+        t.steps = self.steps[:]
+        t.debugging = self.debugging
+        return t
+
+    def debug(self, flag):
+        """t.debug(flag) turns debugging on or off."""
+        self.debugging = flag
+
+    def append(self, cmd, kind):
+        """t.append(cmd, kind) adds a new step at the end."""
+        if type(cmd) is not type(''):
+            raise TypeError, \
+                  'Template.append: cmd must be a string'
+        if kind not in stepkinds:
+            raise ValueError, \
+                  'Template.append: bad kind ' + `kind`
+        if kind == SOURCE:
+            raise ValueError, \
+                  'Template.append: SOURCE can only be prepended'
+        if self.steps and self.steps[-1][1] == SINK:
+            raise ValueError, \
+                  'Template.append: already ends with SINK'
+        if kind[0] == 'f' and not re.search(r'\$IN\b', cmd):
+            raise ValueError, \
+                  'Template.append: missing $IN in cmd'
+        if kind[1] == 'f' and not re.search(r'\$OUT\b', cmd):
+            raise ValueError, \
+                  'Template.append: missing $OUT in cmd'
+        self.steps.append((cmd, kind))
+
+    def prepend(self, cmd, kind):
+        """t.prepend(cmd, kind) adds a new step at the front."""
+        if type(cmd) is not type(''):
+            raise TypeError, \
+                  'Template.prepend: cmd must be a string'
+        if kind not in stepkinds:
+            raise ValueError, \
+                  'Template.prepend: bad kind ' + `kind`
+        if kind == SINK:
+            raise ValueError, \
+                  'Template.prepend: SINK can only be appended'
+        if self.steps and self.steps[0][1] == SOURCE:
+            raise ValueError, \
+                  'Template.prepend: already begins with SOURCE'
+        if kind[0] == 'f' and not re.search(r'\$IN\b', cmd):
+            raise ValueError, \
+                  'Template.prepend: missing $IN in cmd'
+        if kind[1] == 'f' and not re.search(r'\$OUT\b', cmd):
+            raise ValueError, \
+                  'Template.prepend: missing $OUT in cmd'
+        self.steps.insert(0, (cmd, kind))
+
+    def open(self, file, rw):
+        """t.open(file, rw) returns a pipe or file object open for
+        reading or writing; the file is the other end of the pipeline."""
+        if rw == 'r':
+            return self.open_r(file)
+        if rw == 'w':
+            return self.open_w(file)
+        raise ValueError, \
+              'Template.open: rw must be \'r\' or \'w\', not ' + `rw`
+
+    def open_r(self, file):
+        """t.open_r(file) and t.open_w(file) implement
+        t.open(file, 'r') and t.open(file, 'w') respectively."""
+        if not self.steps:
+            return open(file, 'r')
+        if self.steps[-1][1] == SINK:
+            raise ValueError, \
+                  'Template.open_r: pipeline ends width SINK'
+        cmd = self.makepipeline(file, '')
+        return os.popen(cmd, 'r')
+
+    def open_w(self, file):
+        if not self.steps:
+            return open(file, 'w')
+        if self.steps[0][1] == SOURCE:
+            raise ValueError, \
+                  'Template.open_w: pipeline begins with SOURCE'
+        cmd = self.makepipeline('', file)
+        return os.popen(cmd, 'w')
+
+    def copy(self, infile, outfile):
+        return os.system(self.makepipeline(infile, outfile))
+
+    def makepipeline(self, infile, outfile):
+        cmd = makepipeline(infile, self.steps, outfile)
+        if self.debugging:
+            print cmd
+            cmd = 'set -x; ' + cmd
+        return cmd
+
+
+def makepipeline(infile, steps, outfile):
+    # Build a list with for each command:
+    # [input filename or '', command string, kind, output filename or '']
+
+    list = []
+    for cmd, kind in steps:
+        list.append(['', cmd, kind, ''])
+    #
+    # Make sure there is at least one step
+    #
+    if not list:
+        list.append(['', 'cat', '--', ''])
+    #
+    # Take care of the input and output ends
+    #
+    [cmd, kind] = list[0][1:3]
+    if kind[0] == 'f' and not infile:
+        list.insert(0, ['', 'cat', '--', ''])
+    list[0][0] = infile
+    #
+    [cmd, kind] = list[-1][1:3]
+    if kind[1] == 'f' and not outfile:
+        list.append(['', 'cat', '--', ''])
+    list[-1][-1] = outfile
+    #
+    # Invent temporary files to connect stages that need files
+    #
+    garbage = []
+    for i in range(1, len(list)):
+        lkind = list[i-1][2]
+        rkind = list[i][2]
+        if lkind[1] == 'f' or rkind[0] == 'f':
+            temp = tempfile.mktemp()
+            garbage.append(temp)
+            list[i-1][-1] = list[i][0] = temp
+    #
+    for item in list:
+        [inf, cmd, kind, outf] = item
+        if kind[1] == 'f':
+            cmd = 'OUT=' + quote(outf) + '; ' + cmd
+        if kind[0] == 'f':
+            cmd = 'IN=' + quote(inf) + '; ' + cmd
+        if kind[0] == '-' and inf:
+            cmd = cmd + ' <' + quote(inf)
+        if kind[1] == '-' and outf:
+            cmd = cmd + ' >' + quote(outf)
+        item[1] = cmd
+    #
+    cmdlist = list[0][1]
+    for item in list[1:]:
+        [cmd, kind] = item[1:3]
+        if item[0] == '':
+            if 'f' in kind:
+                cmd = '{ ' + cmd + '; }'
+            cmdlist = cmdlist + ' |\n' + cmd
+        else:
+            cmdlist = cmdlist + '\n' + cmd
+    #
+    if garbage:
+        rmcmd = 'rm -f'
+        for file in garbage:
+            rmcmd = rmcmd + ' ' + quote(file)
+        trapcmd = 'trap ' + quote(rmcmd + '; exit') + ' 1 2 3 13 14 15'
+        cmdlist = trapcmd + '\n' + cmdlist + '\n' + rmcmd
+    #
+    return cmdlist
+
+
+# Reliably quote a string as a single argument for /bin/sh
+
+_safechars = string.ascii_letters + string.digits + '!@%_-+=:,./' # Safe unquoted
+_funnychars = '"`$\\'                           # Unsafe inside "double quotes"
+
+def quote(file):
+    for c in file:
+        if c not in _safechars:
+            break
+    else:
+        return file
+    if '\'' not in file:
+        return '\'' + file + '\''
+    res = ''
+    for c in file:
+        if c in _funnychars:
+            c = '\\' + c
+        res = res + c
+    return '"' + res + '"'
+
+
+# Small test program and example
+
+def test():
+    print 'Testing...'
+    t = Template()
+    t.append('togif $IN $OUT', 'ff')
+    t.append('giftoppm', '--')
+    t.append('ppmtogif >$OUT', '-f')
+    t.append('fromgif $IN $OUT', 'ff')
+    t.debug(1)
+    FILE = '/usr/local/images/rgb/rogues/guido.rgb'
+    t.copy(FILE, '@temp')
+    print 'Done.'

data/lib/rubyfox/server/data/lib/Lib/popen2.py

@@ -0,0 +1,304 @@
+"""
+popen2.py
+
+Implement popen2 module functionality for Jython.
+
+Note that the popen* methods in this module follow the return value
+ordering of the Python popen2.popen* methods:
+
+	fromChild, toChild = popen2.popen2(...)
+	fromChild, toChild, errorFromChild = popen2.popen3(...)
+	fromChildInclError, toChild = popen2.popen4(...)
+
+The os.popen* methods are more natural as follows:
+
+	toChild, fromChild = os.popen2(...)
+	toChild, fromChild, errorFromChild = os.popen3(...)
+	toChild, fromChildInclError = os.popen4(...)
+
+The Popen3 and Popen4 classes allow users to poll() or wait() for
+child processes to terminate.
+"""
+
+import jarray
+from java.lang import System
+from java.util import Vector
+from java.io import BufferedOutputStream
+from java.io import BufferedInputStream
+from java.io import PipedOutputStream
+from java.io import PipedInputStream
+from org.python.core import PyFile
+from javashell import shellexecute
+
+__all__ = ["popen", "popen2", "popen3", "popen4", "Popen3", "Popen4"]
+
+_active = []
+
+class _ProcessFile:
+    """Python file object that returns the process exit status from
+    the close method.
+    """
+    def __init__(self, stream, process, name):
+        self._file = PyFile(stream, "'%s'" % name)
+        self._process = process
+
+    def __getattr__(self, name):
+        return getattr(self._file, name)
+
+    def __repr__(self):
+        return `self._file`
+        
+    def close(self):
+        self._file.close()
+        return self._process.waitFor() or None
+
+class Popen3:
+    """Class representing a child process.  Normally instances are created
+    by the factory functions popen2() and popen3()."""
+
+    sts = -1                            # Child not completed yet
+    childWaiter = None
+    count = 0
+    
+    def __init__(self, cmd, capturestderr=0, bufsize=-1):
+        """The parameter 'cmd' is the shell command to execute in a
+        sub-process.  Can be either a sequence of executable
+        and arguments, or a shell command.
+        The 'capturestderr' flag, if true, specifies that
+        the object should capture standard error output of the child process.
+        The default is false.  If the 'bufsize' parameter is specified, it
+        specifies the size of the I/O buffers to/from the child process.
+        """
+        self.process = shellexecute( cmd )
+        self._tochild = self.process.getOutputStream()
+        self._fromchild = self.process.getInputStream()
+        if capturestderr:
+            self._childerr = self.process.getErrorStream()
+        else:
+            self._childerr = None
+        import threading
+        self.childWaiterLock = threading.Lock()
+
+        if bufsize > 0:
+            self._tochild = BufferedOutputStream( self._tochild, bufsize )
+            self._fromchild = BufferedInputStream( self._fromchild, bufsize )
+            if self._childerr:
+                self._childerr = BufferedInputStream(
+                    self._childerr,
+                    bufsize
+                    )
+                
+        self.tochild = PyFile( self._tochild )
+        self.fromchild = PyFile( self._fromchild )
+        if self._childerr:
+            self.childerr = PyFile( self._childerr )
+
+    def _startChildWaiter(self):
+        """Start a subthread that waits for the child process to exit."""
+        self.childWaiterLock.acquire()
+        try:
+            if not self.childWaiter:
+                import threading
+                self.childWaiter = threading.Thread(
+                    target=self.wait,
+                    name="ChildWaiter %s" % self.process,
+                    args=()
+                    )
+                self.childWaiter.setDaemon( 1 )
+                self.childWaiter.start()
+        finally:
+            self.childWaiterLock.release()
+
+    def poll(self):
+        """Return the exit status of the child process if it has finished,
+        or -1 if it hasn't finished yet."""
+        if self.sts < 0 and not self.childWaiter:
+            self._startChildWaiter()
+            self.childWaiter.join( .1 )
+        return self.sts
+
+    def wait(self):
+        """Wait for and return the exit status of the child process."""
+        self.sts = self.process.waitFor()
+        # some processes won't terminate until tochild stream is
+        # closed, but that's really the responsibility of the caller
+        return self.sts
+
+def _makeReaderThread( stream, outfunc, bufsize, name=None, postFunc=None ):
+    """Create a thread that reads the stream, calling outfunc for each block,
+    and calling postFunc when the end of stream is reached.
+    """
+    Popen3.count += 1
+    name = name or str( Popen3.count )
+    threadName = "StreamReader %s" % name
+    import threading
+    reader = threading.Thread(
+        target=_readStream,
+        name=threadName,
+        args=( stream, outfunc, bufsize, postFunc )
+        )
+
+    reader.setDaemon( 1 )
+    reader.start()
+    return reader
+    
+def _readStream( instream, outfunc, bufsize, postFunc=None ):
+    """Read instream, calling outfunc( buf, 0, count ) with each block.
+    Copy streams by passing destStream.write as the outfunc.
+    postFunc is called when the end of instream is reached.
+    """
+    bufsize = bufsize < 1 and 4096 or bufsize
+    buf = jarray.zeros( bufsize, 'b' )
+    total = 0
+    while 1:
+        count = instream.read( buf )
+        if -1 == count:
+            instream.close()
+            if postFunc: postFunc()
+            break
+        else:
+            total += count
+            outfunc( buf, 0, count )
+    return total
+
+class Popen4(Popen3):
+    """Popen object that joins the stdout and stderr streams into a single  
+    output stream."""
+    childerr = None
+
+    def __init__(self, cmd, bufsize=-1):
+        Popen3.__init__( self, cmd, 1, bufsize )
+        self.closed = Vector() # use a vector for synchronization close()
+        self.fromchild = self._join(
+            self._fromchild,
+            self._childerr,
+            bufsize
+            )
+
+    def _join( self, stdout, stderr, bufsize ):
+        """create a stream that joins two output streams"""
+        self._pipeOut = PipedOutputStream()
+        joinedStream = PipedInputStream( self._pipeOut )
+        self._outReader = _makeReaderThread(
+            stdout,
+            self._pipeOut.write,
+            bufsize,
+            "%s-stdout" % self.process,
+            self._close
+            )
+        self._errReader = _makeReaderThread(
+            stderr,
+            self._pipeOut.write,
+            bufsize,
+            "%s-stderr" % self.process,
+            self._close
+            )
+        return PyFile( joinedStream )
+
+    def _close( self ):
+        """Must be closed twice (once for each of the two joined pipes)"""
+        self.closed.add( None )
+        if self.closed.size() > 1:
+            self._pipeOut.close()
+
+def popen(path, mode='r', bufsize=-1):
+    p = Popen3( path, 0, bufsize )
+    if mode == 'r':
+        return _ProcessFile(p.fromchild, p.process, path)
+    elif mode == 'w':
+        return _ProcessFile(p.tochild, p.process, path)
+    else:
+        raise OSError(0, "Invalid mode", mode)
+
+def popen2(path, mode="t", bufsize=-1):
+    p = Popen3(path, 0, bufsize)
+    return p.fromchild, p.tochild
+
+def popen3(path, mode="t", bufsize=-1):
+    p = Popen3(path, 1, bufsize)
+    return p.fromchild, p.tochild, p.childerr
+
+def popen4(path, mode="t", bufsize=-1):
+    p = Popen4(path, bufsize)
+    return p.fromchild, p.tochild
+
+def system( cmd ):
+    """Imitate the standard library 'system' call.
+    Execute 'cmd' in a shell, and send output to stdout & stderr.
+
+    This is in popen2 only because its Jython implementation is similar to
+    that of the popen functions.
+    """
+    bufsize = 4096
+    # this uses some Popen3 internals, and thus belongs in popen3
+    # javaos.system should also be this function
+    p = Popen3( cmd, 1, bufsize)
+    p.tochild.close()
+    
+    # read stderr in separate thread
+    errReader = _makeReaderThread(
+        p._childerr,
+        System.err.write,
+        bufsize,
+        "stderr"
+        )
+
+    # read stdin in main thread
+    _readStream(
+        p._fromchild,
+        System.out.write,
+        bufsize
+        )
+
+    status = p.wait()
+    return status
+
+def _test():
+    # _test comes from python22/lib/popen2.py
+    cmd  = "cat"
+    teststr = "ab cd\n"
+    import os
+    if os.name in [ "nt", "java" ]:
+        cmd = "more"
+    # "more" doesn't act the same way across Windows flavors,
+    # sometimes adding an extra newline at the start or the
+    # end.  So we strip whitespace off both ends for comparison.
+    expected = teststr.strip()
+    print "testing popen2..."
+    r, w = popen2(cmd)
+    w.write(teststr)
+    w.close()
+    got = r.read()
+    if got.strip() != expected:
+        raise ValueError("wrote %s read %s" % (teststr, got))
+    print "testing popen3..."
+    try:
+        r, w, e = popen3([cmd])
+    except:
+        r, w, e = popen3(cmd)
+    w.write(teststr)
+    w.close()
+    got = r.read()
+    err = e.read()
+    if got.strip() != expected:
+        raise ValueError("wrote %s read %s, error %s" % (teststr, got, err ))
+    if err:
+        raise ValueError("unexected %s on stderr" % err )
+# this portion of the test is inapplicable to the Jython implementation
+#    for inst in _active[:]:
+#        inst.wait()
+#    if _active:
+#        raise ValueError("_active not empty")
+    print "All OK"
+
+    p = Popen3( cmd )
+    q = "This is\na test of\nwriting\n"
+    p.tochild.write( q )
+    p.tochild.close()
+    r = p.fromchild.read()
+    x = p.poll()
+    assert x == 0
+    assert r.strip() == q.strip()
+
+if __name__ == '__main__':
+    _test()